From b8dc3646cac038b24987b0148f7ab2dbba12b82c Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Tue, 27 Oct 2026 16:16:55 +0200 Subject: [PATCH] fix(lemonade): resolve registered GGUF models and discover serving models in dash Resolve registered GGUF models from the installed Lemonade catalog instead of falling back to the default model. Simple entries (one checkpoint with a variant, no extra files) are served directly by the selected GPU llama-server; anything else keeps the Lemonade router. On Linux a failed router load or verification falls back to the selected GPU backend's llama-server, never to a missing or CPU backend. Split (sharded) GGUF entries resolve to their first shard, and a complete set wins over an older partial download. Persist the name that is actually served, route complex catalog entries through Lemonade, and restrict lifecycle actions to registry-owned services. A non-ROCm GPU backend (Vulkan) is named in the serve output and log, on both routes, only once the model is actually serving. rocm dash discovers loaded models from a local Lemonade server on Linux and WSL (read-only, configurable host and port, opt-out), groups equivalent endpoints and never adopts an external service. A Lemonade rate is the last completed request's rate: it is labelled, left out of the node-wide tok/s and tok/W figures, and flagged in the agent snapshot. rocm services stop also prints the service id and status for the in-process stop report. Add routing, ownership, telemetry and user-journey regressions, including scenarios for the registered-name routes, external discovery, and the reinstall advice for a missing llama-server. Signed-off-by: Eugene Volen --- README.md | 48 +- apps/rocm/src/cli.rs | 29 + apps/rocm/src/dash.rs | 94 +- apps/rocm/src/engine_runtime_select.rs | 69 +- apps/rocm/src/services_cmd.rs | 27 + apps/rocm/src/services_text.rs | 20 +- crates/rocm-core/src/config.rs | 48 + crates/rocm-dash-collectors/src/docker.rs | 1 + crates/rocm-dash-collectors/src/lemonade.rs | 638 +++- crates/rocm-dash-core/src/efficiency.rs | 63 +- crates/rocm-dash-core/src/metrics.rs | 115 +- crates/rocm-dash-core/src/observation.rs | 58 + crates/rocm-dash-core/src/traits.rs | 19 + crates/rocm-dash-daemon/src/demo.rs | 3 +- crates/rocm-dash-daemon/src/registry.rs | 196 +- crates/rocm-dash-daemon/src/runner.rs | 1542 ++------- .../rocm-dash-daemon/src/runner/discovery.rs | 1306 ++++++++ .../tests/discovery_reconcile.rs | 55 +- .../tests/telemetry_contract.rs | 1392 +++++++- crates/rocm-dash-tui/src/agent/snapshot.rs | 25 +- crates/rocm-dash-tui/src/app/event_loop.rs | 2 + crates/rocm-dash-tui/src/app/mod.rs | 3 + crates/rocm-dash-tui/src/app/quit_confirm.rs | 75 +- crates/rocm-dash-tui/src/app/test_support.rs | 1 + crates/rocm-dash-tui/src/app/types.rs | 4 + crates/rocm-dash-tui/src/ui/format.rs | 24 + .../rocm-dash-tui/src/ui/instance_detail.rs | 145 +- crates/rocm-dash-tui/src/ui/launcher.rs | 1 + crates/rocm-dash-tui/src/ui/mod.rs | 1 + crates/rocm-dash-tui/src/ui/quit_confirm.rs | 60 +- .../rocm-dash-tui/src/ui/services_manager.rs | 308 +- crates/rocm-dash-tui/src/ui/tabs/home.rs | 64 +- crates/rocm-dash-tui/src/ui/tabs/instances.rs | 187 +- crates/rocm-dash-tui/src/ui/tabs/observe.rs | 49 + crates/rocm-dash-tui/src/ui/tabs/pane.rs | 327 +- crates/rocm-dash-tui/src/ui/tabs/serving.rs | 22 +- .../tests/dash_characterization.rs | 6 +- docs/architecture.md | 22 +- docs/manual-testing.md | 110 +- docs/testing.md | 189 +- engines/aim/src/docker.rs | 2 +- engines/lemonade/src/backend_alignment.rs | 15 +- engines/lemonade/src/device_policy.rs | 33 + engines/lemonade/src/direct_llama.rs | 2870 +++++++++++++---- engines/lemonade/src/lib.rs | 1041 ++++-- engines/lemonade/src/llamacpp_backend.rs | 130 +- engines/lemonade/src/process.rs | 323 +- engines/lemonade/src/router_serve.rs | 860 +++++ engines/lemonade/src/runtime_dir.rs | 16 + engines/lemonade/src/state.rs | 20 +- engines/lemonade/src/test_http.rs | 163 + tests/e2e-cucumber/expectations.toml | 10 + tests/e2e-cucumber/features/dash.feature | 64 +- .../features/model_serving.feature | 50 +- .../features/runtime_setup.feature | 8 + tests/e2e-cucumber/src/dash_screen.rs | 301 ++ tests/e2e-cucumber/src/lemonade_backend.rs | 167 + tests/e2e-cucumber/src/lemonade_protocol.rs | 153 + tests/e2e-cucumber/src/lib.rs | 3 + tests/e2e-cucumber/src/serve_log.rs | 31 +- tests/e2e-cucumber/tests/e2e.rs | 64 + tests/e2e-cucumber/tests/e2e/dash_steps.rs | 440 ++- tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 113 + .../tests/e2e/service_stop_steps.rs | 23 +- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 261 +- 65 files changed, 12126 insertions(+), 2383 deletions(-) create mode 100644 crates/rocm-dash-daemon/src/runner/discovery.rs create mode 100644 engines/lemonade/src/router_serve.rs create mode 100644 engines/lemonade/src/test_http.rs create mode 100644 tests/e2e-cucumber/src/dash_screen.rs create mode 100644 tests/e2e-cucumber/src/lemonade_backend.rs create mode 100644 tests/e2e-cucumber/src/lemonade_protocol.rs diff --git a/README.md b/README.md index 143ff6f4c2..c859a2991f 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,33 @@ serving-instance registry needs the Unix domain socket telemetry, so that view is Linux/WSL2 only. Use `rocm dash --demo` for a synthetic session that runs anywhere without a GPU or daemon. +On Linux and WSL, the embedded daemon also discovers loaded models from a +separately running local Lemonade server at `127.0.0.1:13305`. Configure +`dashboard.daemon.lemonade_host` and `dashboard.daemon.lemonade_port` in +`config.json` for another endpoint, or set `dashboard.daemon.enable_lemonade` +to `false` to disable external discovery. The file is `~/.rocm/config.json`, or +`config.json` in the directory named by `ROCM_CLI_CONFIG_DIR`. Editing it is the +only way to set these fields. Native Windows has no live dashboard telemetry, so it +has no external discovery. A healthy server with no loaded models +does not count as a running model. Discovery does not register an external server +as a managed service or give ROCm CLI ownership of its process. External models +are visible in Observe, but are excluded from the Services stop/restart controls. +The tok/s figure for a Lemonade model is the rate of the endpoint's most recent +completed request, not live throughput: it stays the same while the server is +idle. The dashboard explains it with a legend (`Lemonade: last req tok/s, not live`) +under the Instances table and the Services overlay rows, and a `last req` prefix in +the instance detail and the quick preview. Because it is not live, the node-wide +totals (the tok/W and tok/s heroes on Home and Observe) and the per-instance tok/W +leave it out, and the agent snapshot marks it with `gen_tps_last_request`. A managed Lemonade server that requires an API key stays listed with its +registry status and throughput unavailable; the dashboard never reads the key. + +The engine pane distinguishes a ROCm CLI-managed Lemonade installation from an +externally running server. When installation detection is unavailable, it shows +an unchecked status ("unknown, run `rocm engines list`") rather than claiming +Lemonade is not installed. Installation status does not depend on GPU telemetry. +It is read once, when the dashboard starts. After you install Lemonade from the +Engines overlay, restart the dashboard to refresh it. + ## Configure ROCm and serve a model Before serving a model, check whether a managed ROCm runtime is already @@ -1236,7 +1263,26 @@ server instead (an already-serving one is reported at once). Watch the download Which model form to pass depends on the engine your GPU selects. The Lemonade engine (Ryzen AI or Radeon) serves llama.cpp **GGUF** models. Pass a GGUF repo with an explicit quantization variant, for example, -`rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0`. +`rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0`. Lemonade also accepts +registered GGUF names such as `Qwen3-0.6B-GGUF`, or the `qwen-smoke` alias. +On Linux, direct serving resolves simple single-GGUF entries from the installed +Lemonade catalog. Entries needing additional files or model-specific options, +well-formed entries of an unrecognized shape, and registered names on Windows use +Lemonade's router instead. That includes a registered checkpoint that is not a +plain `owner/repo:variant`, such as one whose variant is `*`. Explicit `owner/repo:variant` references require the selected GPU +llama-server backend. Direct serving uses that backend, including Vulkan when +supported; it does not select a stale ROCm binary merely because it is on disk. +When Lemonade selects a GPU backend other than ROCm, such as Vulkan, serving names +that backend instead of presenting it as ROCm. Neither route silently substitutes +the default model. + +On Linux, if Lemonade's router fails to load or verify a registered model, `rocm +serve` stops the router and serves that model directly through the llama-server of +the backend Lemonade selected, when the catalog entry resolves to a simple +checkpoint. The serve log names the failure and the backend. This never uses a CPU +backend: with no GPU llama-server for the selected backend, or on Windows, the +router's error is returned and the service is marked failed. A server that did not +become ready, a failed backend selection and a later server exit are never retried. The vLLM engine (Instinct) serves **safetensors** repos, such as `rocm serve Qwen/Qwen2.5-0.5B-Instruct`. diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 652003072f..ae58817d22 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -692,6 +692,11 @@ rocm logs --search error timeout")] status: bool, }, /// Launch the unified telemetry dashboard (TUI) with an embedded daemon. + /// + /// On Linux/WSL, discovers loaded models from local Lemonade at 127.0.0.1:13305 + /// by default. Configure dashboard.daemon.enable_lemonade, lemonade_host, and + /// lemonade_port in config.json. External models have no Services stop/restart + /// controls; discovery is read-only and quitting leaves external servers running. Dash { /// Replay a recorded session NDJSON instead of connecting to a live daemon. #[arg(long, value_name = "FILE")] @@ -1493,6 +1498,30 @@ mod tests { ); } + #[test] + fn dash_help_scopes_lemonade_discovery_to_linux_and_wsl() { + // The embedded daemon that does the discovery is Unix-only, so the help + // must not promise it on every platform. README.md says the same. + let help = Cli::command() + .find_subcommand_mut("dash") + .expect("dash subcommand") + .render_long_help() + .to_string(); + let flat = help.split_whitespace().collect::>().join(" "); + assert!( + flat.contains("On Linux/WSL, discovers loaded models from local Lemonade"), + "{help}" + ); + // This pins the wording only. Scenario + // `@id:dash-lemonade-external-protocol-discovery` proves the claim: the + // external server receives no request beyond reads after the dashboard + // quits. + assert!( + flat.contains("quitting leaves external servers running"), + "{help}" + ); + } + #[test] fn the_framework_choice_is_offered_on_the_command_line() { // Guards the actual regression: the variants existed in the library and diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 454efc7b38..679ec85650 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -58,10 +58,11 @@ pub fn runner_options( instance_tick: Duration::from_secs_f64(d.instance_tick_secs), disable_vllm_metrics: false, vllm_metrics_host: "127.0.0.1".into(), - // Lemonade discovery stays opt-in (mirrors a no-flag embedded daemon). - enable_lemonade: false, - lemonade_host: "127.0.0.1".into(), - lemonade_port: 13305, + // Read-only discovery of local external Lemonade; managed registry + // discovery remains independent of this switch. + enable_lemonade: d.enable_lemonade, + lemonade_host: d.lemonade_host.clone(), + lemonade_port: d.lemonade_port, persist_dir: Some(paths.telemetry_state_dir()), // D7 seam consumer: managed services from `rocm serve --managed`. services_dir: Some(paths.services_dir()), @@ -258,6 +259,29 @@ fn automation_summaries(config: &RocmCliConfig) -> Vec { .collect() } +/// Read-only managed-engine inventory, independent of GPU telemetry and +/// endpoint liveness. An unavailable detect response means unknown, not absent. +fn lemonade_managed_installed(paths: &AppPaths) -> Option { + let detect = crate::engine_ipc::engine_request::<_, rocm_engine_protocol::DetectResponse>( + Some(paths), + "lemonade", + rocm_engine_protocol::EngineMethod::Detect, + &rocm_engine_protocol::DetectRequest::default(), + ) + .ok(); + managed_installed_from_detect(detect.as_ref()) +} + +/// Pure verdict over a Detect result: `None` when Detect failed (unknown, not +/// absent); otherwise installed only counts when the runtime is managed. A +/// legacy response without `managed_env` is not proof of a managed install. +fn managed_installed_from_detect( + detect: Option<&rocm_engine_protocol::DetectResponse>, +) -> Option { + let detect = detect?; + Some(detect.installed && detect.managed_env == Some(true)) +} + /// Resolve the TUI args from the unified config + environment. /// /// MUST be called on a synchronous thread *before* any tokio runtime is entered: @@ -328,6 +352,7 @@ pub fn resolved_args( // `resolved_args` itself has no opinion, so this always starts `false` // here. launcher_hosted: false, + lemonade_managed_installed: lemonade_managed_installed(paths), } } @@ -715,6 +740,7 @@ fn launcher_serving_instances(paths: &AppPaths) -> Vec) -> rocm_engine_protocol::DetectResponse { + let mut v = serde_json::json!({ + "installed": installed, "env_id": null, "python_version": null, + "torch_version": null, "transformers_version": null, + "available_devices": [], "notes": [], + "capabilities": {"cpu": false, "rocm_gpu": false, "openai_compatible": true, + "tool_calling": false, "quantized_models": "", "reasoning_parser": false} + }); + if let Some(m) = managed_env { + v["managed_env"] = m.into(); + } + serde_json::from_value(v).unwrap() + } + + #[test] + fn managed_installed_verdict_follows_detect_result() { + assert_eq!( + managed_installed_from_detect(Some(&detect(true, Some(true)))), + Some(true) + ); + assert_eq!( + managed_installed_from_detect(Some(&detect(true, Some(false)))), + Some(false) + ); + // Legacy response without the ownership field proves nothing managed. + assert_eq!( + managed_installed_from_detect(Some(&detect(true, None))), + Some(false) + ); + assert_eq!( + managed_installed_from_detect(Some(&detect(false, Some(true)))), + Some(false) + ); + // A failed Detect is unknown, never "not installed". + assert_eq!(managed_installed_from_detect(None), None); + } + + #[test] + fn lemonade_discovery_defaults_and_overrides_reach_daemon() { + let default = runner_options(&RocmCliConfig::default(), &paths(), false); + assert!(default.enable_lemonade); + assert_eq!(default.lemonade_host, "127.0.0.1"); + assert_eq!(default.lemonade_port, 13305); + let config: RocmCliConfig = serde_json::from_str( + r#"{ + "dashboard": {"daemon": {"enable_lemonade": false, + "lemonade_host": "localhost", "lemonade_port": 14567}} + }"#, + ) + .unwrap(); + let overridden = runner_options(&config, &paths(), false); + assert!(!overridden.enable_lemonade); + assert_eq!(overridden.lemonade_host, "localhost"); + assert_eq!(overridden.lemonade_port, 14567); + assert_eq!(overridden.services_dir, Some(paths().services_dir())); + } + fn cfg() -> RocmCliConfig { RocmCliConfig::default() } @@ -1915,6 +1998,9 @@ mod tests { assert_eq!(inst.model_name, "meta-llama/Llama-3.1-8B"); assert_eq!(inst.port, Some(11435)); assert_eq!(inst.status, rocm_dash_core::metrics::InstanceStatus::Ready); + // Registry records are owned by this CLI; the flag must survive the map + // so lifecycle surfaces treat the launcher row as managed. + assert!(inst.managed, "registry-owned record must stay managed"); let _ = std::fs::remove_dir_all(&root); } diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index 3200aaec70..88d1019187 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -18,7 +18,7 @@ //! `service_http_readiness_paths`, `http_get_local_service`, //! `http_post_local_service_json`, `service_http_readiness_response_ready`, //! `lemonade_health_ready_for_model`, `model_list_ready_for_model`, -//! `service_model_gpu_ready`, `service_model_reports_rocm_backend`, +//! `service_model_gpu_ready`, `service_model_reports_gpu_backend`, //! `service_model_names_match`). //! //! Mechanically relocated from `main.rs` with no behavior change (the @@ -27,7 +27,9 @@ //! immediately after it in the original file, `treat_as_natural_language`, is //! dispatch-adjacent natural-language-invocation routing that stays in //! `main.rs` on purpose — it is not part of this module and is not touched by -//! this change. +//! this change. Behavior here has changed since the move: a Lemonade router +//! entry served on Vulkan now counts as ready (`service_model_gpu_ready`), not +//! only one served on ROCm. //! //! One item physically interleaved in this span in `main.rs` stayed there //! instead of moving: `const fn status_for_readiness` sits between @@ -76,7 +78,7 @@ //! `service_http_readiness_paths`, `http_get_local_service`, //! `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, //! `model_list_ready_for_model`, `service_model_gpu_ready`, and -//! `service_model_reports_rocm_backend` have no callers outside this module +//! `service_model_reports_gpu_backend` have no callers outside this module //! and stay private. //! //! The cluster's regression tests live in this module's own `mod tests`, @@ -495,7 +497,7 @@ fn lemonade_health_ready_for_model(body: &str, canonical_model_id: &str) -> bool .into_iter() .filter_map(|field| model.get(field).and_then(serde_json::Value::as_str)) .any(|loaded| service_model_names_match(loaded, canonical_model_id)); - name_matches && service_model_reports_rocm_backend(model) + name_matches && service_model_reports_gpu_backend(model) }) }) } @@ -503,7 +505,7 @@ fn lemonade_health_ready_for_model(body: &str, canonical_model_id: &str) -> bool fn model_list_ready_for_model( body: &str, canonical_model_id: &str, - require_rocm_backend: bool, + require_gpu_backend: bool, ) -> bool { let Ok(value) = serde_json::from_str::(body.trim()) else { return false; @@ -517,7 +519,7 @@ fn model_list_ready_for_model( .into_iter() .filter_map(|field| model.get(field).and_then(serde_json::Value::as_str)) .any(|loaded| service_model_names_match(loaded, canonical_model_id)); - name_matches && (!require_rocm_backend || service_model_gpu_ready(model)) + name_matches && (!require_gpu_backend || service_model_gpu_ready(model)) }) }) } @@ -525,22 +527,26 @@ fn model_list_ready_for_model( /// Whether a lemonade `/v1/models` entry is served on GPU. A stock `llama-server` /// (direct-serve) entry has no `recipe_options` and is accepted (that path only runs /// GPU backends); a Lemonade-router entry carries `recipe_options`, so it must report a -/// ROCm backend — which keeps a registered-but-unloaded model (empty `recipe_options`) +/// GPU backend (ROCm or Vulkan) — which keeps a registered-but-unloaded model (empty `recipe_options`) /// from reading as ready. fn service_model_gpu_ready(model: &serde_json::Value) -> bool { match model.get("recipe_options") { None => true, - Some(_) => service_model_reports_rocm_backend(model), + Some(_) => service_model_reports_gpu_backend(model), } } -fn service_model_reports_rocm_backend(model: &serde_json::Value) -> bool { +fn service_model_reports_gpu_backend(model: &serde_json::Value) -> bool { model .get("recipe_options") .and_then(|options| options.get("llamacpp_backend")) .or_else(|| model.get("llamacpp_backend")) .and_then(serde_json::Value::as_str) - .is_some_and(|backend| backend.trim().to_ascii_lowercase().starts_with("rocm")) + .is_some_and(|backend| { + // The GPU backends the engine can select (ROCm, else Vulkan); never `cpu`. + let backend = backend.trim().to_ascii_lowercase(); + backend.starts_with("rocm") || backend.starts_with("vulkan") + }) } /// Whether two model names refer to the same model, leniently. @@ -725,6 +731,49 @@ mod tests { )); } + #[test] + fn lemonade_router_readiness_accepts_any_gpu_backend_the_engine_selects() { + // The engine selects ROCm first and Vulkan otherwise, and records either as + // `backend_requested`; a router-served model on a Vulkan host must read as + // ready on both probe paths. CPU (or no backend) must never read as ready. + for (backend, ready) in [ + ("rocm", true), + ("rocm-stable", true), + ("vulkan", true), + ("Vulkan", true), + ("cpu", false), + ("", false), + ] { + let health = json!({ + "all_models_loaded": [{ + "model_name": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": backend } + }] + }) + .to_string(); + let models = json!({ + "data": [{ + "id": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": backend } + }] + }) + .to_string(); + for (path, body) in [("/v1/health", &health), ("/v1/models", &models)] { + assert_eq!( + service_http_readiness_response_ready( + "lemonade", + path, + 200, + body, + "Qwen3-0.6B-GGUF" + ), + ready, + "{path} with backend `{backend}`" + ); + } + } + } + #[test] fn serve_readiness_wait_withholds_ready_while_the_model_only_lists() -> Result<()> { use std::io::{Read, Write}; diff --git a/apps/rocm/src/services_cmd.rs b/apps/rocm/src/services_cmd.rs index 79c21675f4..f2cbc52623 100644 --- a/apps/rocm/src/services_cmd.rs +++ b/apps/rocm/src/services_cmd.rs @@ -242,4 +242,31 @@ mod tests { assert!(rendered.contains("status: stopped")); assert!(rendered.contains("stopped processes: 2")); } + + /// The stop report production hands the renderer is the one + /// `stop_internal_managed_service` builds: `service_id` and `status` sit + /// directly on `result`, not under `result.service`. Scenario + /// `@id:serve-lemonade-simple-registered-direct-route` asserts the service id + /// in the stop output, so the renderer must print it for this shape. + #[test] + fn service_stop_render_prints_the_in_process_stop_report_shape() { + let rendered = crate::services_text::render_service_action_result( + "stop_server", + &serde_json::json!({ + "output": { + "result": { + "service_id": "svc-qwen", + "status": "stopped", + "engine_stop": {"attempted": true, "stopped": true, "graceful": true}, + "signaled_pids": [1234] + } + } + }), + ); + + assert!(rendered.contains("Local server stopped")); + assert!(rendered.contains("service: svc-qwen"), "{rendered}"); + assert!(rendered.contains("status: stopped"), "{rendered}"); + assert!(rendered.contains("stopped processes: 1")); + } } diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index fb0e97f29c..cd6af59fb6 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -1029,11 +1029,21 @@ pub(crate) fn render_service_action_result(tool: &str, value: &serde_json::Value return render_unconfirmed_service_stop(output); } let action = service_action_past_tense(tool); - let service = output.get("service").or_else(|| { - output - .get("result") - .and_then(|result| result.get("service")) - }); + // Restart nests the record under `service`; the in-process stop report + // ([`stop_internal_managed_service`]) is itself the record's summary, with + // `service_id` and `status` directly on `result`. + let service = output + .get("service") + .or_else(|| { + output + .get("result") + .and_then(|result| result.get("service")) + }) + .or_else(|| { + output + .get("result") + .filter(|result| result.get("service_id").is_some()) + }); let mut text = String::new(); let _ = writeln!(text, "Local server {action}"); if let Some(service) = service { diff --git a/crates/rocm-core/src/config.rs b/crates/rocm-core/src/config.rs index d6a6cbce20..514fe34a9b 100644 --- a/crates/rocm-core/src/config.rs +++ b/crates/rocm-core/src/config.rs @@ -353,6 +353,18 @@ const fn default_instance_tick_secs() -> f64 { 2.0 } +fn default_dashboard_lemonade_host() -> String { + "127.0.0.1".to_owned() +} + +const fn default_dashboard_lemonade_port() -> u16 { + 13305 +} + +const fn default_dashboard_enable_lemonade() -> bool { + true +} + /// Telemetry daemon operational spec. Tick cadences are stored as f64 seconds in /// the unified JSON config; use the `*_tick()` accessors for `Duration`s. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -369,6 +381,14 @@ pub struct DashboardDaemonConfig { pub discovery_tick_secs: f64, #[serde(default = "default_instance_tick_secs")] pub instance_tick_secs: f64, + /// Discover loaded models on the configured external Lemonade endpoint. + /// Read-only; does not adopt, launch, or stop the external service. + #[serde(default = "default_dashboard_enable_lemonade")] + pub enable_lemonade: bool, + #[serde(default = "default_dashboard_lemonade_host")] + pub lemonade_host: String, + #[serde(default = "default_dashboard_lemonade_port")] + pub lemonade_port: u16, /// Watch this file for new normalized benchmark CSV rows. When unset, the /// daemon derives `/bench/results.csv` from the current `AppPaths` /// at startup so machine-specific paths are never persisted in config. @@ -384,6 +404,9 @@ impl Default for DashboardDaemonConfig { gpu_tick_secs: default_gpu_tick_secs(), discovery_tick_secs: default_discovery_tick_secs(), instance_tick_secs: default_instance_tick_secs(), + enable_lemonade: default_dashboard_enable_lemonade(), + lemonade_host: default_dashboard_lemonade_host(), + lemonade_port: default_dashboard_lemonade_port(), bench_results_dir: None, } } @@ -1201,6 +1224,31 @@ mod tests { } } + #[test] + fn dashboard_daemon_defaults_enable_local_lemonade_discovery() { + let old_config: DashboardDaemonConfig = serde_json::from_str("{}").unwrap(); + let json = serde_json::to_value(old_config).unwrap(); + assert_eq!(json["enable_lemonade"], true); + assert_eq!(json["lemonade_host"], "127.0.0.1"); + assert_eq!(json["lemonade_port"], 13305); + } + + #[test] + fn dashboard_daemon_lemonade_config_overrides_round_trip() { + let config: DashboardDaemonConfig = serde_json::from_str( + r#"{ + "enable_lemonade": false, "lemonade_host": "localhost", "lemonade_port": 14567 + }"#, + ) + .unwrap(); + assert!(!config.enable_lemonade); + assert_eq!(config.lemonade_host, "localhost"); + assert_eq!(config.lemonade_port, 14567); + let restored: DashboardDaemonConfig = + serde_json::from_value(serde_json::to_value(&config).unwrap()).unwrap(); + assert_eq!(restored, config); + } + #[test] fn dashboard_daemon_tick_accessors_map_secs_to_duration() { let d = DashboardDaemonConfig { diff --git a/crates/rocm-dash-collectors/src/docker.rs b/crates/rocm-dash-collectors/src/docker.rs index 8e573dcbfd..9bea9d5841 100644 --- a/crates/rocm-dash-collectors/src/docker.rs +++ b/crates/rocm-dash-collectors/src/docker.rs @@ -294,6 +294,7 @@ fn parse_container(inspect: &ContainerInspectResponse) -> Result>, } -/// `/api/v1/health` response — liveness + the most-recently-loaded model name. +/// Lemonade health response. `all_models_loaded`, when present, is the +/// authoritative loaded set. Older servers report only `model_loaded`. #[derive(Debug, Clone, Default, Deserialize)] pub struct LemonadeHealth { #[serde(default)] pub status: Option, #[serde(default)] pub model_loaded: Option, + #[serde(default)] + pub all_models_loaded: Option>, +} + +impl LemonadeHealth { + /// Sorted, deduplicated loaded model names, only from a healthy response. + /// An empty router is healthy but does not constitute a serving model. + pub fn loaded_models(&self) -> Vec { + if self.status.as_deref() != Some("ok") { + return Vec::new(); + } + let names: Vec<&str> = match &self.all_models_loaded { + Some(models) => models + .iter() + .filter_map(|model| { + model.as_str().or_else(|| { + ["model_name", "id", "name"] + .into_iter() + .filter_map(|field| { + model.get(field).and_then(serde_json::Value::as_str) + }) + .find(|name| !name.trim().is_empty()) + }) + }) + .collect(), + None => self.model_loaded.as_deref().into_iter().collect(), + }; + names + .into_iter() + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(str::to_owned) + .collect::>() + .into_iter() + .collect() + } + + /// Singleton attribution for server-global last-request stats. Health's + /// `model_loaded` means last access (load OR inference), not last request, + /// so it cannot disambiguate multiple loaded models. A conflicting explicit + /// name also rejects the singleton fallback. + pub fn stats_model(&self) -> Option { + let loaded = self.loaded_models(); + if loaded.len() != 1 { + return None; + } + if let Some(last) = self + .model_loaded + .as_deref() + .map(str::trim) + .filter(|name| !name.is_empty()) + { + return loaded.into_iter().find(|name| name == last); + } + loaded.into_iter().next() + } +} + +/// Map a failed request to a [`CollectorError`]. A refused or unanswered +/// connection becomes `Io(ConnectionRefused)`, the typed marker for "nothing is +/// listening" that [`is_connect_error`] reads; every other failure stays a +/// `Transport` error. A timeout counts as unanswered because Windows does not +/// refuse a connection to a closed local port: the attempt hangs until the +/// request timeout. +fn send_error(url: &str, error: &reqwest::Error) -> CollectorError { + let message = format!("GET {url}: {error}"); + if error.is_connect() || error.is_timeout() { + CollectorError::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionRefused, + message, + )) + } else { + CollectorError::Transport(message) + } +} + +/// True when `error` means no server accepted the connection, as opposed to a +/// server that answered with an HTTP status or a broken body. +pub fn is_connect_error(error: &CollectorError) -> bool { + matches!(error, CollectorError::Io(io) if io.kind() == std::io::ErrorKind::ConnectionRefused) } /// PURE: parse a `/api/v1/stats` body into an [`InstanceSample`]. /// -/// Lemonade reports an instantaneous `tokens_per_second` rate (mapped to `gen_tps`) rather than a +/// Lemonade reports the `tokens_per_second` rate of the most recent request (mapped to `gen_tps`; +/// it is not a live rate: the same value is reported until another request completes, so the +/// dashboard keeps showing the last request's rate until a new request replaces it) rather than a /// cumulative counter; it exposes no KV-cache / running / waiting metrics, so /// those stay `None`. Malformed/empty JSON → an all-`None` default, never panics. pub fn parse_stats(body: &str) -> InstanceSample { @@ -90,11 +172,18 @@ pub fn parse_stats_struct(body: &str) -> LemonadeStats { object_or_default(body) } -/// PURE: extract the loaded model name from a `/api/v1/health` body, if present -/// and non-empty. Malformed JSON → `None`, never panics. -pub fn parse_health_model(body: &str) -> Option { - let health: LemonadeHealth = object_or_default(body); - health.model_loaded.filter(|s| !s.is_empty()) +/// PURE: parse a health body, rejecting invalid/wrong-shape input to default. +pub fn parse_health(body: &str) -> LemonadeHealth { + let Ok(value @ serde_json::Value::Object(_)) = serde_json::from_str(body) else { + return LemonadeHealth::default(); + }; + if value + .get("all_models_loaded") + .is_some_and(|models| !models.is_array() && !models.is_null()) + { + return LemonadeHealth::default(); + } + serde_json::from_value(value).unwrap_or_default() } /// Deserialize a JSON **object** body into `T`, returning `T::default()` for any @@ -116,22 +205,27 @@ pub fn lemonade_container_id(host: &str, port: u16) -> String { format!("lemonade-{host}-{port}") } -/// PURE: build a `DiscoveredService` for a Lemonade endpoint. Lemonade is a -/// single local server (no tensor-parallel sharding metadata), so TP = 1. +/// A model identity within an external endpoint. Encoding the model's UTF-8 +/// bytes avoids separator collisions and keeps identity stable across reorder. +pub fn lemonade_model_id(host: &str, port: u16, model_name: &str) -> String { + use std::fmt::Write as _; + let mut id = format!("{}-model-", lemonade_container_id(host, port)); + for byte in model_name.as_bytes() { + let _ = write!(id, "{byte:02x}"); + } + id +} + +/// PURE: build metadata for an external loaded model. /// -/// Every call site only constructs this after `/api/v1/health` has already -/// answered (see `LemonadeCollector::discover`), so a successfully-built -/// service is by definition reachable and serving — mark it `Ready` rather -/// than the generic `Running`. +/// Callers must establish healthy loaded-model evidence first; answering HTTP +/// alone is insufficient. `model_name` must be a trimmed, non-empty name, as +/// produced by [`LemonadeHealth::loaded_models`]. Lemonade exposes no TP metadata. pub fn lemonade_service(host: &str, port: u16, model_name: &str) -> DiscoveredService { DiscoveredService { - container_id: lemonade_container_id(host, port), + container_id: lemonade_model_id(host, port, model_name), container_name: "lemonade".to_string(), - model_name: if model_name.is_empty() { - "lemonade".to_string() - } else { - model_name.to_string() - }, + model_name: model_name.to_string(), port: Some(port), tensor_parallel_size: 1, status: rocm_dash_core::metrics::InstanceStatus::Ready, @@ -139,18 +233,23 @@ pub fn lemonade_service(host: &str, port: u16, model_name: &str) -> DiscoveredSe } } -/// PURE: build a finished `Instance` from a Lemonade `/health` model name + a `/stats` body. -/// -/// This is the fixture→Instance anchor (no network). `gen_tps` flows from -/// `tokens_per_second`; KV/req fields stay `None` (Lemonade does not report them). -pub fn lemonade_instance(host: &str, port: u16, model_name: &str, stats_body: &str) -> Instance { - let svc = lemonade_service(host, port, model_name); - let sample = parse_stats(stats_body); - merge_instance(&svc, &sample, 0, 0) +/// PURE: map a health payload to one discovered service per loaded model. +/// Invalid, unhealthy or empty payloads yield no rows. The daemon runner and +/// the tests both use this single mapping. +pub fn services_from_health( + host: &str, + port: u16, + health: &LemonadeHealth, +) -> Vec { + health + .loaded_models() + .iter() + .map(|model| lemonade_service(host, port, model)) + .collect() } /// Async scrape of a local Lemonade endpoint. Network/parse failure degrades to -/// `Err`/`None` ("not reachable") — never a panic. The pure parsers above do the +/// transport errors or an empty loaded set, never a panic. The pure parsers above do the /// actual mapping; this only does I/O. #[derive(Debug, Clone)] pub struct LemonadeCollector { @@ -173,17 +272,47 @@ impl LemonadeCollector { } fn url(&self, path: &str) -> String { - format!("http://{}:{}{path}", self.host, self.port) + let host = if self.host.contains(':') && !self.host.starts_with('[') { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + format!("http://{host}:{}{path}", self.port) } - async fn get_text(&self, path: &str) -> Result { - let url = self.url(path); - let resp = self + /// GET `path`, falling back to the legacy route on a 404. `Ok(None)` means + /// neither route exists: the server does not speak the Lemonade protocol. + async fn get_text_opt(&self, path: &str, auth_optional: bool) -> Result> { + let mut url = self.url(path); + let mut resp = self .client .get(&url) .send() .await - .map_err(|e| CollectorError::Transport(format!("GET {url}: {e}")))?; + .map_err(|e| send_error(&url, &e))?; + // Router versions expose either /api/v1 or /v1. Only a missing route + // warrants compatibility fallback, never an authentication or health + // failure. This does not add a llama-server metrics protocol. + if resp.status() == reqwest::StatusCode::NOT_FOUND + && let Some(legacy_path) = path.strip_prefix("/api") + { + url = self.url(legacy_path); + resp = self + .client + .get(&url) + .send() + .await + .map_err(|e| send_error(&url, &e))?; + } + if resp.status() == reqwest::StatusCode::NOT_FOUND + || (auth_optional + && matches!( + resp.status(), + reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN + )) + { + return Ok(None); + } if !resp.status().is_success() { return Err(CollectorError::Transport(format!( "GET {url}: status {}", @@ -192,40 +321,76 @@ impl LemonadeCollector { } resp.text() .await + .map(Some) .map_err(|e| CollectorError::Transport(format!("body {url}: {e}"))) } + async fn get_text(&self, path: &str) -> Result { + self.get_text_opt(path, false).await?.ok_or_else(|| { + CollectorError::Transport(format!("GET {}: status 404 Not Found", self.url(path))) + }) + } + /// Scrape `/api/v1/stats` → an `InstanceSample` (rate → `gen_tps`). pub async fn fetch_stats(&self) -> Result { Ok(parse_stats(&self.get_text(LEMONADE_STATS_PATH).await?)) } - /// Scrape `/api/v1/health` → the loaded model name (if any). - pub async fn fetch_health_model(&self) -> Result> { - Ok(parse_health_model( - &self.get_text(LEMONADE_HEALTH_PATH).await?, - )) + /// Registry-target lookup: `Ok(None)` if the Lemonade health route is absent + /// or requires credentials (401/403). The registry still supplies ownership + /// and status; this does not establish health or permit a stats scrape. + /// Transport failures and 5xx responses are still errors. External discovery + /// uses [`Self::fetch_health`] instead, where auth rejection remains an error. + pub async fn fetch_health_if_router(&self) -> Result> { + Ok(self + .get_text_opt(LEMONADE_HEALTH_PATH, true) + .await? + .map(|body| parse_health(&body))) } - /// Probe the endpoint: if `/api/v1/health` answers, return a `DiscoveredService` - /// for it (model name from health, falling back to "lemonade"); otherwise - /// `None` (endpoint absent/unreachable) — a clean no-op, no panic. - pub async fn discover(&self) -> Option { - match self.fetch_health_model().await { - Ok(model) => Some(lemonade_service( - &self.host, - self.port, - model.as_deref().unwrap_or(""), - )), - Err(_) => None, - } + /// Fetch the health payload. Invalid payloads carry no loaded models. + pub async fn fetch_health(&self) -> Result { + Ok(parse_health(&self.get_text(LEMONADE_HEALTH_PATH).await?)) } } #[cfg(test)] mod tests { + /// The dashboard config's default Lemonade port and this crate's constant are + /// two literals in two crates (this crate depends on `rocm-core`, not the + /// reverse), so nothing but this test keeps them equal. + #[test] + fn the_dashboard_default_port_is_the_collectors_lemonade_port() { + assert_eq!( + super::LEMONADE_PORT, + rocm_core::DashboardDaemonConfig::default().lemonade_port + ); + } + use super::*; + /// An IPv6 host is bracketed in the URL exactly once; IPv4 and names are not. + /// Pure, so it runs on hosts where the daemon's IPv6 tests cannot bind. + #[test] + fn url_brackets_an_ipv6_host_once() { + let url = |host: &str| { + LemonadeCollector::new(host, 13305, std::time::Duration::from_secs(1)) + .url("/api/v1/health") + }; + assert_eq!(url("::1"), "http://[::1]:13305/api/v1/health"); + assert_eq!(url("[::1]"), "http://[::1]:13305/api/v1/health"); + assert_eq!(url("127.0.0.1"), "http://127.0.0.1:13305/api/v1/health"); + assert_eq!(url("localhost"), "http://localhost:13305/api/v1/health"); + } + + /// Same path the daemon runner takes: health fetch, then the shared mapping. + async fn discovered(collector: &LemonadeCollector) -> Vec { + match collector.fetch_health().await { + Ok(health) => services_from_health(&collector.host, collector.port, &health), + Err(_) => Vec::new(), + } + } + // Representative `/api/v1/stats` response (lemonade-server.ai/docs/api). const STATS_FIXTURE: &str = r#"{ "time_to_first_token": 2.14, @@ -241,9 +406,149 @@ mod tests { "status": "ok", "version": "9.3.3", "model_loaded": "Llama-3.2-1B-Instruct-Hybrid", - "all_models_loaded": [] + "all_models_loaded": [{"model_name":"Llama-3.2-1B-Instruct-Hybrid"}] }"#; + #[test] + fn loaded_set_and_stats_attribution_are_conservative() { + let health = parse_health( + r#"{"status":"ok","all_models_loaded":[" Model-B ",{"id":"Model-A"},{"name":"Model-B"},{"model_name":"","id":"Model-C"},null,7,{}]}"#, + ); + assert_eq!(health.loaded_models(), ["Model-A", "Model-B", "Model-C"]); + assert_eq!(health.stats_model(), None); + let health = parse_health( + r#"{"status":"ok","model_loaded":"Model-B","all_models_loaded":["Model-A","Model-B"]}"#, + ); + assert_eq!( + health.stats_model(), + None, + "last access can be a load, not a request" + ); + for body in [ + r#"{"status":"ok","all_models_loaded":["Model-A"]}"#, + r#"{"status":"ok","model_loaded":"Model-A"}"#, + ] { + assert_eq!(parse_health(body).stats_model().as_deref(), Some("Model-A")); + } + assert_eq!( + parse_health( + r#"{"status":"ok","model_loaded":"Model-B","all_models_loaded":["Model-A"]}"# + ) + .stats_model(), + None + ); + // Names are compared exactly, never case-folded: the server treats `Model-A` + // and `model-a` as two models, so they are two rows, and a last-request rate + // whose name differs only in case is not attributed to either. + let health = parse_health(r#"{"status":"ok","all_models_loaded":["Model-A","model-a"]}"#); + assert_eq!(health.loaded_models(), ["Model-A", "model-a"]); + assert_eq!(health.stats_model(), None); + assert_eq!( + parse_health( + r#"{"status":"ok","model_loaded":"model-a","all_models_loaded":["Model-A"]}"# + ) + .stats_model(), + None + ); + for body in [ + r#"{"status":"ok","all_models_loaded":{},"model_loaded":"Model-A"}"#, + r#"{"status":"bad","model_loaded":"Model-A"}"#, + r#"{"model_loaded":"Model-A"}"#, + ] { + assert!(parse_health(body).loaded_models().is_empty()); + } + } + + #[tokio::test] + async fn health_if_router_distinguishes_missing_route_from_failure() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + // A plain llama-server: neither Lemonade health route exists. + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/health")) + .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"status":"ok"}"#)) + .mount(&server) + .await; + let collector = + LemonadeCollector::new("127.0.0.1", server.address().port(), Duration::from_secs(1)); + assert!( + collector + .fetch_health_if_router() + .await + .expect("missing routes are not an error") + .is_none() + ); + assert!(collector.fetch_health().await.is_err()); + // A router answering on the legacy route is still a router. + server.reset().await; + Mock::given(method("GET")) + .and(path("/v1/health")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(r#"{"status":"ok","model_loaded":"Model-A"}"#), + ) + .mount(&server) + .await; + let health = collector.fetch_health_if_router().await.unwrap().unwrap(); + assert_eq!(health.loaded_models(), ["Model-A"]); + // Only registry lookup tolerates auth rejection, without claiming health. + for status in [401, 403] { + server.reset().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(status)) + .mount(&server) + .await; + assert!(collector.fetch_health_if_router().await.unwrap().is_none()); + assert!(collector.fetch_health().await.is_err()); + } + // Unrelated server failures must still warn. + for status in [500, 503] { + server.reset().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(status)) + .mount(&server) + .await; + assert!( + collector.fetch_health_if_router().await.is_err(), + "status {status} must stay an error" + ); + } + } + + #[tokio::test] + async fn health_http_failure_never_falls_back_to_successful_route() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + for status in [401, 403, 500, 503] { + server.reset().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(status)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/health")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(r#"{"status":"ok","model_loaded":"Model-A"}"#), + ) + .expect(0) + .mount(&server) + .await; + let collector = LemonadeCollector::new( + "127.0.0.1", + server.address().port(), + Duration::from_secs(1), + ); + assert!(discovered(&collector).await.is_empty()); + server.verify().await; + } + } + #[test] fn parse_stats_maps_tokens_per_second_to_gen_tps() { let sample = parse_stats(STATS_FIXTURE); @@ -267,13 +572,30 @@ mod tests { } #[test] - fn parse_health_model_extracts_loaded_model() { + fn parse_health_extracts_loaded_model() { assert_eq!( - parse_health_model(HEALTH_FIXTURE).as_deref(), + parse_health(HEALTH_FIXTURE).stats_model().as_deref(), Some("Llama-3.2-1B-Instruct-Hybrid") ); } + #[test] + fn a_null_model_list_keeps_the_rest_of_the_health_body() { + // An unset optional field is an explicit `null` for FastAPI/Pydantic + // servers; the body is otherwise healthy and must not be discarded. + let health = + parse_health(r#"{"status":"ok","model_loaded":"Model-A","all_models_loaded":null}"#); + assert_eq!(health.status.as_deref(), Some("ok")); + assert_eq!(health.loaded_models().len(), 1); + // A present but wrong-shape list is still rejected whole. + for body in [ + r#"{"status":"ok","all_models_loaded":{}}"#, + r#"{"status":"ok","all_models_loaded":"x"}"#, + ] { + assert_eq!(parse_health(body).status, None, "body {body}"); + } + } + #[test] fn malformed_or_empty_input_is_graceful_not_panic() { // Garbage / empty / wrong-shape → all-None sample, no panic. @@ -288,19 +610,27 @@ mod tests { assert_eq!(sample.gen_tps, None, "body {body:?}"); assert!(sample.kv_cache_usage_pct.is_none()); } - assert_eq!(parse_health_model("not json"), None); - assert_eq!(parse_health_model(r#"{"model_loaded": ""}"#), None); + assert!(parse_health("not json").loaded_models().is_empty()); + assert!( + parse_health(r#"{"model_loaded": ""}"#) + .loaded_models() + .is_empty() + ); } #[test] fn fixture_stats_plus_health_becomes_an_instance() { // The fixture→Instance anchor: no network. Health gives the model name, // /stats gives the live rate → an Instances-tab row. - let model = parse_health_model(HEALTH_FIXTURE).expect("model"); - let inst = lemonade_instance("127.0.0.1", LEMONADE_PORT, &model, STATS_FIXTURE); + let model = parse_health(HEALTH_FIXTURE).stats_model().expect("model"); + let svc = lemonade_service("127.0.0.1", LEMONADE_PORT, &model); + let inst = rocm_dash_core::traits::merge_instance(&svc, &parse_stats(STATS_FIXTURE), 0, 0); assert_eq!(inst.model_name, "Llama-3.2-1B-Instruct-Hybrid"); assert_eq!(inst.container_name, "lemonade"); - assert_eq!(inst.container_id, "lemonade-127.0.0.1-13305"); + assert_eq!( + inst.container_id, + lemonade_model_id("127.0.0.1", 13305, &model) + ); assert_eq!(inst.port, Some(13305)); assert_eq!(inst.gen_tps, Some(33.33)); assert_eq!(inst.status, rocm_dash_core::metrics::InstanceStatus::Ready); @@ -310,14 +640,177 @@ mod tests { } #[tokio::test] - async fn discover_is_clean_noop_when_endpoint_absent() { - // Probe a port with no server → connection refused → None, no panic. - // (Port 0 is never a live listener; the GET fails fast.) - let collector = LemonadeCollector::new("127.0.0.1", 1, Duration::from_millis(200)); - assert!(collector.discover().await.is_none()); + async fn discovery_rejects_invalid_health_and_healthy_empty_router() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + let collector = + LemonadeCollector::new("127.0.0.1", server.address().port(), Duration::from_secs(1)); + for body in [ + "not json", + "{}", + "[]", + "null", + r#"{"status":"error","model_loaded":"Model-A"}"#, + r#"{"status":"ok","all_models_loaded":[]}"#, + r#"{"status":"ok","all_models_loaded":[],"model_loaded":"Old-Model"}"#, + r#"{"status":"ok","model_loaded":" "}"#, + ] { + server.reset().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(200).set_body_string(body)) + .mount(&server) + .await; + assert!( + discovered(&collector).await.is_empty(), + "health {body:?} must not advertise a serving model" + ); + } + } + + #[tokio::test] + async fn discovery_recognizes_modern_loaded_model_without_legacy_field() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(200).set_body_string( + r#"{"status":"ok","all_models_loaded":[{"model_name":"Model-A"}]}"#, + )) + .mount(&server) + .await; + let collector = + LemonadeCollector::new("127.0.0.1", server.address().port(), Duration::from_secs(1)); + assert_eq!(discovered(&collector).await[0].model_name, "Model-A"); + } + + #[tokio::test] + async fn discovery_accepts_v1_health_only_when_api_prefix_is_absent() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v1/health")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(r#"{"status":"ok","model_loaded":"Model-A"}"#), + ) + .mount(&server) + .await; + let collector = + LemonadeCollector::new("127.0.0.1", server.address().port(), Duration::from_secs(1)); + assert_eq!(discovered(&collector).await[0].model_name, "Model-A"); + } + + #[tokio::test] + async fn v1_fallback_status_failure_names_the_requested_v1_url() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/health")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + let port = server.address().port(); + let collector = LemonadeCollector::new("127.0.0.1", port, Duration::from_secs(1)); + let error = collector.fetch_health().await.unwrap_err().to_string(); + let wanted = format!("http://127.0.0.1:{port}/v1/health"); + assert!(error.contains(&wanted), "{error}"); + assert!(!error.contains("/api/v1/health"), "{error}"); + } + + #[tokio::test] + async fn v1_fallback_transport_failure_names_host_port_and_path() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + // 404 on the primary route, then the retry is cut off by the client + // timeout so it fails at the transport level. + Mock::given(method("GET")) + .and(path(LEMONADE_HEALTH_PATH)) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/v1/health")) + .respond_with(ResponseTemplate::new(200).set_delay(Duration::from_secs(2))) + .mount(&server) + .await; + let port = server.address().port(); + let collector = LemonadeCollector::new("127.0.0.1", port, Duration::from_millis(200)); + let error = collector.fetch_health().await.unwrap_err().to_string(); + let wanted = format!("GET http://127.0.0.1:{port}/v1/health"); + assert!(error.contains(&wanted), "{error}"); + } + + #[tokio::test] + async fn health_and_stats_return_transport_errors_when_endpoint_absent() { + // Reserve then release a random port: an absent listener yields health + // and stats transport errors, never a panic. + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let collector = LemonadeCollector::new("127.0.0.1", port, Duration::from_millis(200)); + assert!(collector.fetch_health().await.is_err()); assert!(collector.fetch_stats().await.is_err()); } + /// Windows does not refuse a connection to a closed local port: the attempt + /// hangs until the request timeout. Both outcomes mean nothing answered, so + /// both must read as "absent" and not as a broken server. + #[tokio::test] + async fn closed_and_unanswered_endpoints_both_read_as_nothing_listening() { + let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let closed_port = closed.local_addr().unwrap().port(); + drop(closed); + let refused = LemonadeCollector::new("127.0.0.1", closed_port, Duration::from_millis(200)) + .fetch_health() + .await + .unwrap_err(); + assert!(is_connect_error(&refused), "{refused}"); + + // Accepts the connection at the kernel level but never answers. + let silent = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let silent_port = silent.local_addr().unwrap().port(); + let timed_out = + LemonadeCollector::new("127.0.0.1", silent_port, Duration::from_millis(200)) + .fetch_health() + .await + .unwrap_err(); + assert!(is_connect_error(&timed_out), "{timed_out}"); + drop(silent); + } + + /// An HTTP status error is a server that answered, so it is not "absent". + #[tokio::test] + async fn an_answering_server_with_an_error_status_is_not_nothing_listening() { + use wiremock::matchers::method; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + let error = + LemonadeCollector::new("127.0.0.1", server.address().port(), Duration::from_secs(1)) + .fetch_health() + .await + .unwrap_err(); + assert!(!is_connect_error(&error), "{error}"); + } + /// Integration-gated: scrape a REAL local Lemonade server (install the engine /// with `rocm engines install lemonade` and start it first). Not run in CI. #[tokio::test] @@ -325,7 +818,8 @@ mod tests { async fn live_lemonade_scrape_produces_instance() { let collector = LemonadeCollector::new("127.0.0.1", LEMONADE_PORT, Duration::from_millis(1500)); - let svc = collector.discover().await.expect("lemonade reachable"); + let svcs = discovered(&collector).await; + let svc = svcs.first().expect("lemonade reachable"); assert_eq!(svc.port, Some(LEMONADE_PORT)); // A stats scrape returns a sample (gen_tps may be None until an inference). let _sample = collector.fetch_stats().await.expect("stats scrape"); diff --git a/crates/rocm-dash-core/src/efficiency.rs b/crates/rocm-dash-core/src/efficiency.rs index ded34043c6..9a6d19fa6d 100644 --- a/crates/rocm-dash-core/src/efficiency.rs +++ b/crates/rocm-dash-core/src/efficiency.rs @@ -126,7 +126,8 @@ pub fn attribute_power_w( /// Whole-node efficiency: aggregate generation throughput over aggregate power. #[derive(Debug, Clone, Copy, PartialEq)] pub struct NodeEfficiency { - /// Σ `gen_tps` ÷ Σ board `power_w`, in tokens per watt. + /// Σ live `gen_tps` (see [`Instance::live_gen_tps`]) ÷ Σ board `power_w`, in + /// tokens per watt. pub tokens_per_watt: f64, /// Freshness of the throughput behind the figure (instances at zero tok/s /// contribute nothing and are not counted): `Held` when any of it is @@ -150,8 +151,9 @@ impl NodeEfficiency { /// Aggregate generation throughput across instances. #[derive(Debug, Clone, Copy, PartialEq)] pub struct NodeThroughput { - /// Σ `gen_tps` over the instances with a finite reading, in tokens/s. A real - /// zero counts: an idle node is zero throughput, not no data. + /// Σ `gen_tps` over the instances with a finite, live reading (a Lemonade + /// last-request rate is not live), in tokens/s. A real zero counts: an idle + /// node is zero throughput, not no data. pub tokens_per_sec: f64, /// Freshness of the throughput behind the sum, as for /// [`NodeEfficiency::freshness`]: instances at zero tok/s add nothing to the @@ -167,8 +169,8 @@ impl NodeThroughput { } } -/// Node throughput over `instances`, or `None` when no instance has a finite -/// `gen_tps` (a single NaN/Inf must not poison the sum). +/// Node throughput over `instances`, or `None` when no instance has a finite, +/// live `gen_tps` (a single NaN/Inf must not poison the sum). /// /// This is the one place the node's throughput and its freshness are worked /// out, so every surface that shows the aggregate (the efficiency hero, the @@ -176,6 +178,11 @@ impl NodeThroughput { /// instance at zero tok/s contributes nothing to the sum, so its freshness says /// nothing about the sum's either: a held idle instance must not mark a figure /// that comes from fresh ones as held. +/// +/// Only live throughput counts: a Lemonade last-request rate (see +/// [`Instance::live_gen_tps`]) stays unchanged while the server is idle, so it is +/// left out of the sum and out of the freshness, and an idle Lemonade server on +/// its own is no throughput reading at all. #[must_use] pub fn node_throughput<'a>( instances: impl IntoIterator, @@ -185,7 +192,7 @@ pub fn node_throughput<'a>( let mut held = false; let mut unknown = false; for inst in instances { - let Some(v) = inst.gen_tps.filter(|v| v.is_finite()) else { + let Some(v) = inst.live_gen_tps() else { continue; }; tokens_per_sec += v; @@ -269,6 +276,16 @@ pub fn tokens_per_watt(gen_tps: Option, power_w: Option) -> Option 0.0).then(|| tps / power) } +/// An instance's efficiency, from its live throughput only. +/// +/// A last-request rate over the board's current (idle) power is not an +/// efficiency, so it yields `None` and never reaches the snapshot, the wire or a +/// recording. +#[must_use] +pub fn instance_tokens_per_watt(inst: &Instance, power_w: Option) -> Option { + tokens_per_watt(inst.live_gen_tps(), power_w) +} + #[cfg(test)] mod tests { use super::*; @@ -315,6 +332,40 @@ mod tests { } } + /// An idle Lemonade server keeps reporting its last request's rate. Counted as + /// throughput it would hold the node figure up long after traffic stopped, and + /// mark it held while it lasted. + #[test] + fn node_efficiency_leaves_out_a_last_request_rate() { + let mut idle = serving("lemonade", Some(500.0), Some(ObservationFreshness::Held)); + idle.gen_tps_last_request = true; + let live = serving("vllm", Some(30.0), Some(ObservationFreshness::Fresh)); + let node = node_efficiency(&snap_with( + vec![idle.clone(), live], + vec![gpu("gpu-0", 100.0)], + )) + .expect("the live rate is traffic"); + // Only the live 30 tok/s counts: 30 / 100 W, not 530 / 100 W. + assert!((node.tokens_per_watt - 0.3).abs() < 1e-9, "{node:?}"); + assert!( + !node.held(), + "a held last-request rate must not mark the figure held" + ); + // An idle Lemonade server alone is no traffic at all. + assert_eq!( + node_efficiency(&snap_with(vec![idle], vec![gpu("gpu-0", 100.0)])), + None + ); + } + + #[test] + fn an_instance_efficiency_leaves_out_a_last_request_rate() { + let mut inst = serving("lemonade", Some(40.0), None); + assert_eq!(instance_tokens_per_watt(&inst, Some(100.0)), Some(0.4)); + inst.gen_tps_last_request = true; + assert_eq!(instance_tokens_per_watt(&inst, Some(100.0)), None); + } + #[test] fn node_efficiency_divides_total_throughput_by_total_board_power() { let snap = snap_with( diff --git a/crates/rocm-dash-core/src/metrics.rs b/crates/rocm-dash-core/src/metrics.rs index c474b1f9eb..691d01ff45 100644 --- a/crates/rocm-dash-core/src/metrics.rs +++ b/crates/rocm-dash-core/src/metrics.rs @@ -61,7 +61,6 @@ pub struct GpuSystemInfo { pub memory_partition_mode: MemoryPartitionMode, pub compute_partition_mode: ComputePartitionMode, pub vram_per_logical_gpu_mb: u64, - pub lemond_version: Option, pub llama_server_build: Option, pub ccr_version: Option, pub llamacpp_backend: Option, @@ -197,6 +196,10 @@ pub struct ObservationMetadata { #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct Instance { + /// Lifecycle ownership proven by discovery from a managed-service record. + /// Missing ownership in legacy snapshots is not permission to mutate. + #[serde(default)] + pub managed: bool, pub container_id: String, pub container_name: String, pub status: InstanceStatus, @@ -220,6 +223,11 @@ pub struct Instance { /// Omitted from JSON when `None` to preserve wire back-compat. #[serde(default, skip_serializing_if = "Option::is_none")] pub gen_tps_observation: Option, + /// True when [`Instance::gen_tps`] is the endpoint's most recent completed + /// request rate (Lemonade `/stats`), which stays unchanged while the server is + /// idle, rather than a live counter rate. Omitted from JSON when false. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub gen_tps_last_request: bool, /// Efficiency: `gen_tps` ÷ the power (W) attributed to this instance /// ([`Instance::power_w`]). `None` when throughput or GPU power telemetry /// is unavailable. @@ -255,6 +263,35 @@ pub struct Instance { pub log_file: Option, } +impl Instance { + /// External Lemonade discovery uses the engine marker, not the instance id. + /// This identifies provenance independently of the current serving status. + #[must_use] + pub fn is_external_lemonade(&self) -> bool { + !self.managed && self.container_name == "lemonade" + } + + /// The finite throughput that is current, for figures that sum instances. + /// + /// A rate that is only the endpoint's last completed request + /// ([`Instance::gen_tps_last_request`]) is excluded: it stays unchanged while + /// the server is idle, so adding it to a node total would show an idle server + /// as still generating. The Instances views show such a rate, labelled, and + /// so do the Services overlay rows and its quick preview. + #[must_use] + pub fn live_gen_tps(&self) -> Option { + self.gen_tps + .filter(|v| v.is_finite() && !self.gen_tps_last_request) + } + + /// [`Instance::tokens_per_watt`] when it derives from live throughput; `None` + /// when it derives from a last-request rate (see [`Instance::live_gen_tps`]). + #[must_use] + pub fn live_tokens_per_watt(&self) -> Option { + self.tokens_per_watt.filter(|_| !self.gen_tps_last_request) + } +} + #[cfg(test)] mod tests { use super::*; @@ -272,6 +309,10 @@ mod tests { "log_file": null }"#; let inst: Instance = serde_json::from_str(legacy).expect("legacy instance must parse"); + assert!( + !inst.managed, + "legacy snapshots do not grant lifecycle control" + ); assert_eq!(inst.gen_tps, None); assert_eq!(inst.tokens_per_watt, None); assert_eq!(inst.ttft_ms, None); @@ -293,6 +334,67 @@ mod tests { assert!((g.power_w - 5.0).abs() < f32::EPSILON); } + #[test] + fn instance_ownership_round_trips_and_defaults_to_false() { + assert!(!Instance::default().managed); + for managed in [false, true] { + let inst = Instance { + managed, + ..Instance::default() + }; + let json = serde_json::to_value(&inst).expect("serialize"); + assert_eq!(json["managed"], managed); + let back: Instance = serde_json::from_value(json).expect("deserialize"); + assert_eq!(back.managed, managed); + } + } + + #[test] + fn a_last_request_rate_is_not_live_throughput_or_live_efficiency() { + let live = Instance { + gen_tps: Some(40.0), + tokens_per_watt: Some(0.5), + ..Instance::default() + }; + assert_eq!(live.live_gen_tps(), Some(40.0)); + assert_eq!(live.live_tokens_per_watt(), Some(0.5)); + let last = Instance { + gen_tps_last_request: true, + ..live + }; + assert_eq!(last.live_gen_tps(), None); + assert_eq!(last.live_tokens_per_watt(), None); + // The rate itself is still there for views that label it. + assert_eq!(last.gen_tps, Some(40.0)); + for bad in [f64::NAN, f64::INFINITY] { + let inst = Instance { + gen_tps: Some(bad), + ..Instance::default() + }; + assert_eq!(inst.live_gen_tps(), None); + } + } + + #[test] + fn external_lemonade_predicate_ignores_ids_and_requires_engine_marker() { + for id in ["svc-looks-managed", "lemonade-looks-external"] { + for managed in [false, true] { + for marker in ["lemonade", "vllm", ""] { + let inst = Instance { + managed, + container_id: id.into(), + container_name: marker.into(), + ..Instance::default() + }; + assert_eq!( + inst.is_external_lemonade(), + !managed && marker == "lemonade" + ); + } + } + } + } + #[test] fn instance_round_trips_ttft_tpot_through_serde() { // The new live-latency fields survive a serialize → deserialize cycle. @@ -389,6 +491,17 @@ mod tests { assert_eq!(InstanceStatus::Ready.label(), "READY"); } + #[test] + fn old_recordings_with_a_retired_lemond_version_key_still_load() { + // Recorded sessions written before the field was removed still carry + // the key. This guards loading them (unknown keys are ignored); it does + // not prove the field's removal, which the compiler already enforces. + let mut value = serde_json::to_value(GpuSystemInfo::default()).expect("serialize"); + value["lemond_version"] = serde_json::json!("10.4.0"); + let info: GpuSystemInfo = serde_json::from_value(value).expect("deserialize old snapshot"); + assert_eq!(info.llama_server_build, None); + } + #[test] fn instance_status_starting_round_trips_through_serde() { // The data-carrying variant serializes as an externally-tagged object diff --git a/crates/rocm-dash-core/src/observation.rs b/crates/rocm-dash-core/src/observation.rs index 11361e3c4c..bf96cd70b6 100644 --- a/crates/rocm-dash-core/src/observation.rs +++ b/crates/rocm-dash-core/src/observation.rs @@ -132,6 +132,11 @@ pub struct GenerationObservationTracker { /// `current_value` is `None`. Set only on successful rate computation or /// valid direct-rate injection; never set on missing/transient data. last_observed_at: Option>, + /// True while the held value came from [`Self::observe_direct`]: an + /// endpoint-reported rate of the most recent request, not a live counter rate. + /// Cleared when a counter rate replaces the value, and with the value on + /// counter reset, expiry and `invalidate`. + direct: bool, } impl GenerationObservationTracker { @@ -150,9 +155,16 @@ impl GenerationObservationTracker { samples: VecDeque::with_capacity(capacity.min(16)), current_value: None, last_observed_at: None, + direct: false, } } + /// Whether the rate comes from an endpoint-reported last-request figure + /// ([`Self::observe_direct`]) rather than a cumulative counter. + pub const fn is_direct(&self) -> bool { + self.direct + } + /// The rolling rate-computation window. pub const fn rate_window(&self) -> Duration { self.rate_window @@ -192,6 +204,7 @@ impl GenerationObservationTracker { self.samples.clear(); self.current_value = None; self.last_observed_at = None; + self.direct = false; // New baseline; no rate until second sample. self.samples.push_back((observed_at, total)); return; @@ -231,6 +244,9 @@ impl GenerationObservationTracker { if rate.is_finite() && rate >= 0.0 { self.current_value = Some(rate); self.last_observed_at = Some(observed_at); + // The value is a counter rate now, not an endpoint-reported + // last-request rate, so `is_direct` must report false. + self.direct = false; } } // dt_ms == 0: identical timestamps should not reach here (guard @@ -252,6 +268,7 @@ impl GenerationObservationTracker { } self.current_value = Some(rate); self.last_observed_at = Some(observed_at); + self.direct = true; } /// Return the current rate and observation metadata for the given `now`. @@ -280,6 +297,7 @@ impl GenerationObservationTracker { self.samples.clear(); self.current_value = None; self.last_observed_at = None; + self.direct = false; return (None, None); } @@ -309,6 +327,7 @@ impl GenerationObservationTracker { self.samples.clear(); self.current_value = None; self.last_observed_at = None; + self.direct = false; } } @@ -899,6 +918,45 @@ mod tests { ); } + #[test] + fn a_counter_rate_supersedes_a_direct_rate_and_clears_the_direct_flag() { + let mut tracker = GenerationObservationTracker::new(Duration::from_secs(2)); + assert!(!tracker.is_direct()); + tracker.observe_direct(99.0, t(0)); + assert!(tracker.is_direct()); + // A lone counter baseline computes no rate, so the direct value stands. + tracker.observe_counter(0.0, t(1)); + assert!(tracker.is_direct()); + tracker.observe_counter(50.0, t(6)); + assert!(!tracker.is_direct(), "the value is a counter rate now"); + } + + #[test] + fn every_path_that_clears_the_rate_also_clears_the_direct_flag() { + // Expiry: the daemon labels the rate "last req" from is_direct alone, so a + // flag that outlives its value renders "last req -" instead of "gen -". + let mut tracker = GenerationObservationTracker::new(Duration::from_secs(2)); + tracker.observe_direct(42.0, t(0)); + assert!(tracker.snapshot(t(5)).0.is_some()); + assert!(tracker.is_direct(), "a live direct rate keeps its label"); + assert!(tracker.snapshot(t(12)).0.is_none()); + assert!(!tracker.is_direct(), "expired"); + + let mut tracker = GenerationObservationTracker::new(Duration::from_secs(2)); + tracker.observe_direct(42.0, t(0)); + tracker.invalidate(); + assert!(!tracker.is_direct(), "invalidated"); + + // Counter reset: a baseline leaves the direct value standing, then a lower + // total clears it. + let mut tracker = GenerationObservationTracker::new(Duration::from_secs(2)); + tracker.observe_direct(42.0, t(0)); + tracker.observe_counter(100.0, t(1)); + tracker.observe_counter(10.0, t(2)); + assert!(tracker.snapshot(t(2)).0.is_none()); + assert!(!tracker.is_direct(), "counter reset"); + } + // ── counter→direct→counter interleaving (M3) ───────────────────────────── #[test] diff --git a/crates/rocm-dash-core/src/traits.rs b/crates/rocm-dash-core/src/traits.rs index 3719b91ec8..25e70c798c 100644 --- a/crates/rocm-dash-core/src/traits.rs +++ b/crates/rocm-dash-core/src/traits.rs @@ -53,6 +53,9 @@ pub trait GpuCollector: Send + Sync { /// A discovered serving process or container. #[derive(Debug, Clone, Default)] pub struct DiscoveredService { + /// True only for discovery from a managed-service registry record. + /// External protocol and container discovery do not grant lifecycle control. + pub managed: bool, pub container_id: String, pub container_name: String, pub model_name: String, @@ -136,6 +139,7 @@ pub fn merge_instance( vram_total_mb: u64, ) -> Instance { Instance { + managed: svc.managed, container_id: svc.container_id.clone(), container_name: svc.container_name.clone(), status: svc.status, @@ -158,6 +162,7 @@ pub fn merge_instance( // Freshness provenance: unknown at merge time; set by the runner when // a counter-rate delta is computed. gen_tps_observation: None, + gen_tps_last_request: false, tokens_per_watt: None, power_w: None, // Derived at the runner by windowing the sample's cumulative histogram @@ -176,6 +181,20 @@ pub fn merge_instance( mod tests { use super::*; + #[test] + fn merge_instance_preserves_explicit_ownership() { + assert!(!DiscoveredService::default().managed); + for managed in [false, true] { + let svc = DiscoveredService { + managed, + container_id: "lemonade-misleading-id".into(), + ..Default::default() + }; + let inst = merge_instance(&svc, &InstanceSample::default(), 0, 0); + assert_eq!(inst.managed, managed); + } + } + #[test] fn merge_instance_propagates_quantization() { let svc = DiscoveredService { diff --git a/crates/rocm-dash-daemon/src/demo.rs b/crates/rocm-dash-daemon/src/demo.rs index acb7b6565c..906161474b 100644 --- a/crates/rocm-dash-daemon/src/demo.rs +++ b/crates/rocm-dash-daemon/src/demo.rs @@ -558,7 +558,6 @@ fn build_sysinfo() -> GpuSystemInfo { memory_partition_mode: MemoryPartitionMode::Nps1, compute_partition_mode: ComputePartitionMode::Spx, vram_per_logical_gpu_mb: VRAM_TOTAL_MB, - lemond_version: None, llama_server_build: None, ccr_version: None, llamacpp_backend: None, @@ -614,6 +613,7 @@ fn build_instances(t_s: f64, seed: &mut u64) -> Vec { }; Instance { + managed: false, container_id: c.container_id.into(), container_name: c.container_name.into(), status: InstanceStatus::Running, @@ -630,6 +630,7 @@ fn build_instances(t_s: f64, seed: &mut u64) -> Vec { waiting_reqs: Some(waiting), gen_tps: Some(gen_tps), gen_tps_observation: None, + gen_tps_last_request: false, tokens_per_watt: None, power_w: None, ttft_ms, diff --git a/crates/rocm-dash-daemon/src/registry.rs b/crates/rocm-dash-daemon/src/registry.rs index cf39320c33..2ec6364888 100644 --- a/crates/rocm-dash-daemon/src/registry.rs +++ b/crates/rocm-dash-daemon/src/registry.rs @@ -23,12 +23,15 @@ //! (`EngineKind::default_port`, `docker.rs`, `lemonade.rs`) remain only as a //! fallback for *unmanaged/external* discovery. //! -//! **Co-located scope (known limitation):** scrape targets carry only a port; -//! the runner's vLLM collector scrapes `opts.vllm_metrics_host` (default -//! `127.0.0.1`), so `record.host` is honored only for the co-located daemon -//! (the same assumption the Docker-discovery path already makes). A managed -//! service on a non-loopback host is surfaced but its metrics are scraped at the -//! local address. Per-host scrape targeting is tracked as a D7 follow-up. +//! **Lifecycle ownership:** only this registry seam marks discovered services +//! as managed. External protocol and container discovery remain read-only; +//! instance ids and engine markers are not evidence of lifecycle ownership. +//! +//! **Host scope:** managed Lemonade telemetry honors `record.host` and port. +//! The vLLM collector still scrapes `opts.vllm_metrics_host` (default +//! `127.0.0.1`) at the record's port, so a non-loopback vLLM service is surfaced +//! but its metrics use the configured collector host. External Lemonade +//! discovery uses its own configured host and port. //! //! **A registry that cannot be read in full (trade-off):** a record file that //! exists but cannot be opened (a root-owned file left by `sudo rocm serve`, a @@ -350,6 +353,7 @@ pub fn discovered_from_record(record: &ServiceRecord) -> Option, pub seen: HashSet, pub non_vllm: HashSet, + /// Read-only Lemonade telemetry targets at registry-owned endpoints. + pub lemonade: Vec, /// How many records were skipped because they are no longer running /// (`failed`/`stopped`/unrecognized status). These are dropped from the /// scrape set on purpose - polling a dead port is pointless - but they are @@ -395,15 +401,91 @@ pub struct ManagedDiscovery { pub retired: HashSet, } +/// A managed Lemonade endpoint, including both display and canonical names so +/// a CLI alias does not prevent safe last-request attribution. +#[derive(Debug, Clone)] +pub struct ManagedLemonadeTarget { + pub service_id: String, + pub host: String, + pub port: u16, + pub model_ref: String, + pub canonical_model_id: String, +} + +/// The loopback families a bind host is reachable through. `::` is a +/// dual-stack wildcard (both loopbacks); `::1` is IPv6-only; `localhost`, +/// empty, `127.0.0.1` and `0.0.0.0` are IPv4-only. No DNS is resolved. +enum HostReach { + Loopback { v4: bool, v6: bool }, + Other(String), +} + +fn normalized_host(host: &str) -> String { + host.trim() + .trim_start_matches('[') + .trim_end_matches(']') + .to_ascii_lowercase() +} + +fn host_reach(host: &str) -> HostReach { + let host = normalized_host(host); + match host.as_str() { + "" | "localhost" | "127.0.0.1" | "0.0.0.0" => HostReach::Loopback { + v4: true, + v6: false, + }, + "::" => HostReach::Loopback { v4: true, v6: true }, + "::1" => HostReach::Loopback { + v4: false, + v6: true, + }, + _ => HostReach::Other(host), + } +} + +/// Whether two co-located HTTP bindings address the same service. Wildcard +/// binds are reachable through loopback (`::` through both families); unrelated +/// remote hosts are not aliases. +pub fn endpoints_match(left_host: &str, left_port: u16, right_host: &str, right_port: u16) -> bool { + if left_port != right_port { + return false; + } + match (host_reach(left_host), host_reach(right_host)) { + (HostReach::Loopback { v4: l4, v6: l6 }, HostReach::Loopback { v4: r4, v6: r6 }) => { + (l4 && r4) || (l6 && r6) + } + (HostReach::Other(left), HostReach::Other(right)) => left == right, + _ => false, + } +} + +/// The host a collector should dial for a registry bind host: wildcards are not +/// valid destinations on every platform, so `::` is reached through `::1`. +pub fn scrape_host(host: &str) -> String { + match normalized_host(host).as_str() { + "" | "localhost" | "0.0.0.0" => "127.0.0.1".to_string(), + "::" | "::1" => "::1".to_string(), + host => host.to_string(), + } +} + +/// Whether a collector cached for `cached_host` may serve `target_host`. +/// +/// `endpoints_match` is deliberately non-transitive (`::` covers `127.0.0.1`), so +/// a match alone is not enough: the cached collector dials a fixed address, and +/// reusing it for a target in another address family probes the wrong listener. +pub fn cached_collector_reusable(cached_host: &str, target_host: &str) -> bool { + scrape_host(cached_host) == scrape_host(target_host) +} + /// Convert the loaded registry records into a [`ManagedDiscovery`] — the pure core of the daemon's managed-service discovery tick. /// /// Non-scrapeable records (bad status / port 0) are dropped; vLLM-engine services flow to the /// Prometheus scrape, others are flagged in `non_vllm`. /// -/// NOTE: gen_tps for managed **non-vLLM** engines (e.g. Lemonade) is not yet -/// wired — they appear in the dashboard but are excluded from the vLLM scrape -/// and not routed to a per-engine collector here. vLLM (the Phase-2 acceptance -/// target) is fully wired. Non-vLLM managed scraping is a D7 follow-up. +/// Managed Lemonade endpoints are routed to the existing Lemonade collector. +/// Servers without that protocol (including direct llama-server) keep their +/// registry row, with throughput unavailable rather than fabricated. pub fn discover_managed_services(records: &[ServiceRecord]) -> ManagedDiscovery { let mut out = ManagedDiscovery::default(); for record in records { @@ -416,6 +498,23 @@ pub fn discover_managed_services(records: &[ServiceRecord]) -> ManagedDiscovery } continue; }; + if engine_kind_for(record) == Some(EngineKind::Lemonade) { + // Keep a `::` bind as `::`: it is dual-stack, which only + // `endpoints_match` can see. The runner dials it via `scrape_host`. + let host = match record.host.trim() { + "" | "localhost" | "0.0.0.0" => "127.0.0.1".to_string(), + "[::]" => "::".to_string(), + "[::1]" => "::1".to_string(), + host => host.to_string(), + }; + out.lemonade.push(ManagedLemonadeTarget { + service_id: svc.container_id.clone(), + host, + port: record.port, + model_ref: record.model_ref.clone(), + canonical_model_id: record.canonical_model_id.clone(), + }); + } out.seen.insert(svc.container_id.clone()); out.roots.extend(record.ancestry_roots()); if engine_kind_for(record) != Some(EngineKind::Vllm) { @@ -434,6 +533,82 @@ mod tests { use std::path::PathBuf; use std::time::Duration; + #[test] + fn endpoint_matching_does_not_conflate_distinct_ports_hosts_or_ip_families() { + let same = [ + ("localhost", "127.0.0.1"), + ("0.0.0.0", "127.0.0.1"), + ("", "127.0.0.1"), + ("[::]", "::1"), + ("::", "::1"), + ("::", "127.0.0.1"), + ("[::]", "localhost"), + ("::", "0.0.0.0"), + ("::", "::"), + ("::1", "[::1]"), + ("Remote.Example", "remote.example"), + ]; + for (l, r) in same { + assert!(endpoints_match(l, 13305, r, 13305), "{l} ~ {r}"); + assert!(endpoints_match(r, 13305, l, 13305), "{r} ~ {l}"); + } + let distinct = [ + ("::1", "127.0.0.1"), + ("::1", "localhost"), + ("::1", "0.0.0.0"), + ("::1", ""), + ("::", "remote.example"), + ("127.0.0.1", "remote.example"), + ("remote.example", "other.example"), + ]; + for (l, r) in distinct { + assert!(!endpoints_match(l, 13305, r, 13305), "{l} !~ {r}"); + assert!(!endpoints_match(r, 13305, l, 13305), "{r} !~ {l}"); + } + assert!(!endpoints_match("::", 13306, "::", 13305)); + assert!(!endpoints_match("127.0.0.1", 13306, "127.0.0.1", 13305)); + } + + #[test] + fn cached_collector_is_reused_only_when_it_dials_the_same_host() { + // Same dialed address: reuse across spelling differences. + for (cached, target) in [ + ("127.0.0.1", "localhost"), + ("0.0.0.0", "127.0.0.1"), + ("::", "[::]"), + ("::1", "::"), + ("remote.example", "remote.example"), + ] { + assert!( + cached_collector_reusable(cached, target), + "{cached} -> {target}" + ); + } + // `endpoints_match` accepts these, but the cached collector would dial + // the other address family. + for (cached, target) in [("::", "127.0.0.1"), ("127.0.0.1", "::"), ("::", "0.0.0.0")] { + assert!(endpoints_match(cached, 13305, target, 13305)); + assert!( + !cached_collector_reusable(cached, target), + "{cached} -> {target}" + ); + } + } + + #[test] + fn dual_stack_bind_keeps_its_host_and_is_dialed_through_ipv6_loopback() { + let record: ServiceRecord = serde_json::from_str( + r#"{"service_id":"s","engine":"lemonade","model_ref":"m","host":"::","port":13305,"status":"ready"}"#, + ) + .unwrap(); + let disc = discover_managed_services(&[record]); + assert_eq!(disc.lemonade[0].host, "::"); + assert_eq!(scrape_host("::"), "::1"); + assert_eq!(scrape_host("[::]"), "::1"); + assert_eq!(scrape_host("0.0.0.0"), "127.0.0.1"); + assert_eq!(scrape_host("10.0.0.5"), "10.0.0.5"); + } + fn at(secs: i64) -> DateTime { Utc.timestamp_opt(secs, 0).unwrap() } @@ -784,6 +959,7 @@ mod tests { .unwrap(); let records = load_service_records(&dir); let svc = discovered_from_record(&records[0]).expect("live record converts"); + assert!(svc.managed, "registry discovery proves lifecycle ownership"); assert_eq!(svc.port, Some(8123)); assert_eq!(svc.container_id, "svc-hot"); assert_eq!(svc.model_name, "meta-llama/Llama-3.1-8B"); diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 855c109742..68b2f297e3 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -10,6 +10,13 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::Duration; +mod discovery; + +use discovery::{ + DiscoverySource, DiscoveryState, Reconcile, TelemetryState, registry_read_warning, + vllm_scrape_targets, +}; + use crate::bench_ring::BenchRing; use crate::persist::SessionWriter; use crate::snapshot_ring::SnapshotRing; @@ -39,6 +46,43 @@ use tracing::{info, trace, warn}; /// Refresh GPU system info (versions, partition modes) every N seconds. const SYSINFO_REFRESH_SECS: u64 = 30; +/// Row identity and accepted model names for singleton Lemonade attribution. +#[derive(Clone)] +struct LemonadeRow { + id: String, + model_names: Vec, +} + +#[derive(Clone)] +struct LemonadeEndpoint { + host: String, + port: u16, + collector: LemonadeCollector, + rows: Vec, + /// The endpoint is the external probe (not a registry-managed target). This + /// decides how strictly health is read: an external server that starts to + /// require credentials is a failure to report, a managed target that does not + /// serve the Lemonade protocol is not. It is a property of the endpoint, not + /// of the tick. + external: bool, + /// This tick is a discovery tick: rows are rebuilt from health. External + /// endpoints also appear on scrape-only ticks, with this `false`. + discover_external: bool, +} + +impl LemonadeEndpoint { + fn warning(&self, phase: &str, error: &impl std::fmt::Display) -> String { + let mut ids: Vec<&str> = self.rows.iter().map(|row| row.id.as_str()).collect(); + ids.sort_unstable(); + format!( + "lemonade {phase} {}:{} [{}]: {error}", + self.host, + self.port, + ids.join(", ") + ) + } +} + /// Options for `run_loop`. Mirrors daemon CLI flags + config. #[derive(Debug, Clone)] pub struct RunnerOptions { @@ -58,9 +102,10 @@ pub struct RunnerOptions { pub disable_vllm_metrics: bool, /// Hostname to scrape; `127.0.0.1` for the typical co-located daemon. pub vllm_metrics_host: String, - /// Opt-in probe-based Lemonade discovery: when enabled, probe a local - /// Lemonade endpoint each discovery tick and surface it as an Instance. - /// Off by default so hosts with no Lemonade server never poll a dead port. + /// Probe-based Lemonade discovery: when enabled, probe a local Lemonade + /// endpoint each discovery tick and surface its healthy loaded models. + /// `rocm dash` passes the dashboard config value, which defaults to on; + /// only `RunnerOptions::default()` is off, for standalone use and tests. pub enable_lemonade: bool, /// Lemonade endpoint host + port (defaults `127.0.0.1:13305`). pub lemonade_host: String, @@ -72,7 +117,7 @@ pub struct RunnerOptions { /// When set, the daemon reads `ManagedServiceRecord`s each discovery tick and /// surfaces live ones as scrape targets (port from the registry), so a model /// served via `rocm serve` appears in the dashboard with live `gen_tps` - /// without Docker discovery. Off by default. + /// without Docker discovery. `None` by default; `rocm dash` sets it. pub services_dir: Option, /// Path or command name for the `amd-smi` binary. The managed ROCm SDK /// ships it inside the runtime wheel's bin directory rather than on `PATH`, @@ -318,8 +363,9 @@ pub async fn run_loop( None }; - // Opt-in Lemonade discovery (probe-based; off unless a Lemonade endpoint is - // configured). Distinct from Docker/vLLM discovery — a local server, not a + // Probe-based Lemonade discovery, gated on `opts.enable_lemonade` (the + // dashboard config default is on; `RunnerOptions::default()` is off). + // Distinct from Docker/vLLM discovery — a local server, not a // container — so it is tracked separately from the Docker known set. let lemonade = if opts.enable_lemonade { info!( @@ -335,6 +381,8 @@ pub async fn run_loop( } else { None }; + // Persistent endpoint collectors share their reqwest connection pools across cycles. + let mut managed_lemonade_collectors: HashMap<(String, u16), LemonadeCollector> = HashMap::new(); let mut bench = opts.bench_csv.as_ref().map(|p| { info!(path = %p.display(), "tailing benchmark CSV"); CsvBenchTailer::new(p.clone()) @@ -407,6 +455,7 @@ pub async fn run_loop( let mut docker_warn: Option = None; let mut registry_warn: Option = None; let mut scrape_warns: Vec = Vec::new(); + let mut lemonade_discovery_warn: Option = None; loop { ticker.tick().await; @@ -577,16 +626,6 @@ pub async fn run_loop( } } - // Lemonade discovery — probe the endpoint on the discovery cadence; add a - // Lemonade Instance when reachable, emit Gone when it disappears, and stay - // a clean no-op (no warning/panic) when no endpoint is configured. - if let Some(l) = lemonade.as_ref() - && discovery_due - { - let probed = l.discover().await; - reconcile.reconcile_lemonade(probed, &mut discovery.lemonade_id); - } - // Managed-service registry discovery — read the rocm-cli // `serve` records and surface live ones as scrape targets. The port is // the registry's authority; non-vLLM engines are tracked so the vLLM @@ -611,6 +650,13 @@ pub async fn run_loop( let disc = crate::registry::discover_managed_services(&read.records); discovery.managed_non_vllm = disc.non_vllm; discovery.managed_roots = disc.roots; + let previous = + std::mem::replace(&mut discovery.managed_lemonade, disc.lemonade); + reconcile.reset_moved_lemonade_targets( + &disc.svcs, + &previous, + &discovery.managed_lemonade, + ); reconcile.reconcile_set( DiscoverySource::Managed, &disc.svcs, @@ -624,7 +670,13 @@ pub async fn run_loop( // The records that did read still add and refresh their // services; nothing is retired, because the one that did // not read could be any of the known ones. - let disc = crate::registry::discover_managed_services(&e.partial.records); + let mut disc = crate::registry::discover_managed_services(&e.partial.records); + let previous = discovery.keep_lemonade_through_partial_registry(&mut disc); + reconcile.reset_moved_lemonade_targets( + &disc.svcs, + &previous, + &discovery.managed_lemonade, + ); reconcile.upsert_set( DiscoverySource::Managed, &disc.svcs, @@ -639,9 +691,220 @@ pub async fn run_loop( scrape_warns.clear(); } - // Per-instance vLLM metric scrape, parallel, on its own cadence. The - // Lemonade instance (if any) is scraped via its own collector below, so - // exclude it from the vLLM Prometheus targets. Managed non-vLLM services + // Independent endpoints run concurrently, but health always precedes + // stats within each endpoint. Last-request stats have no model identity: + // only a singleton loaded set matching exactly one row can own the rate. + let scrape_now = tick_count.is_multiple_of(instance_ticks); + if discovery_due || scrape_now { + let managed_endpoint = discovery.managed_lemonade.iter().any(|target| { + crate::registry::endpoints_match( + &target.host, + target.port, + &opts.lemonade_host, + opts.lemonade_port, + ) + }); + let mut endpoints = Vec::new(); + if let Some(collector) = lemonade.as_ref() { + if managed_endpoint && discovery_due { + // Registry binding wins even if its configured model differs + // from health. Never attach external controls to that binding. + Reconcile { + state: &mut runner.state, + telemetry: &mut telemetry, + tx: &tx, + persist: persist.as_ref(), + } + .reconcile_lemonade(&[], &mut discovery.lemonade_ids); + lemonade_discovery_warn = None; + } else if !managed_endpoint && (discovery_due || !discovery.lemonade_ids.is_empty()) + { + endpoints.push(LemonadeEndpoint { + host: opts.lemonade_host.clone(), + port: opts.lemonade_port, + collector: collector.clone(), + rows: discovery + .lemonade_ids + .iter() + .filter_map(|id| { + runner.state.instances.get(id).map(|inst| LemonadeRow { + id: id.clone(), + model_names: vec![inst.model_name.clone()], + }) + }) + .collect(), + external: true, + discover_external: discovery_due, + }); + } + } + let mut groups: HashMap<(String, u16), Vec> = HashMap::new(); + for target in &discovery.managed_lemonade { + // Use the registry's endpoint equivalence without rewriting the + // selected request host or conflating IPv4 and IPv6 listeners. + // Prefer the cached representative so reorder keeps its client. + // A cached collector dials a fixed address, so it is reused + // only when it would dial the same host as this target. + let endpoint = groups + .keys() + .find(|(host, port)| { + crate::registry::endpoints_match(host, *port, &target.host, target.port) + && crate::registry::cached_collector_reusable(host, &target.host) + }) + .or_else(|| { + managed_lemonade_collectors.keys().find(|(host, port)| { + crate::registry::endpoints_match(host, *port, &target.host, target.port) + && crate::registry::cached_collector_reusable(host, &target.host) + }) + }) + .cloned() + .unwrap_or_else(|| (target.host.clone(), target.port)); + groups.entry(endpoint).or_default().push(LemonadeRow { + id: target.service_id.clone(), + model_names: vec![target.model_ref.clone(), target.canonical_model_id.clone()], + }); + } + managed_lemonade_collectors.retain(|endpoint, _| groups.contains_key(endpoint)); + if scrape_now { + for ((host, port), rows) in groups { + let collector = managed_lemonade_collectors + .entry((host.clone(), port)) + .or_insert_with(|| { + LemonadeCollector::new( + crate::registry::scrape_host(&host), + port, + Duration::from_millis(1500), + ) + }) + .clone(); + endpoints.push(LemonadeEndpoint { + host, + port, + collector, + rows, + external: false, + discover_external: false, + }); + } + } + let results = parallel_scrape(endpoints, move |endpoint| async move { + // A managed row can be a server without the Lemonade protocol + // (a direct llama-server). It stays listed with throughput + // unavailable and no standing warning; real failures still warn. + let health = if endpoint.external { + endpoint.collector.fetch_health().await? + } else if let Some(health) = endpoint.collector.fetch_health_if_router().await? { + health + } else { + return Ok((None, endpoint.rows.clone(), None, None)); + }; + let discovered = endpoint.discover_external.then(|| { + rocm_dash_collectors::lemonade::services_from_health( + &endpoint.host, + endpoint.port, + &health, + ) + }); + let rows = discovered.as_ref().map_or_else( + || endpoint.rows.clone(), + |services| { + services + .iter() + .map(|service| LemonadeRow { + id: service.container_id.clone(), + model_names: vec![service.model_name.clone()], + }) + .collect() + }, + ); + let attribution = health.stats_model(); + let mut matching = rows.iter().filter(|row| { + attribution + .as_ref() + .is_some_and(|model| row.model_names.iter().any(|name| name == model)) + }); + let attributed_id = matching + .next() + .filter(|_| matching.next().is_none()) + .map(|row| row.id.clone()); + // The stats ride with the id they belong to, so a fetch cannot + // exist without the row that attribution picked. + let stats = match attributed_id.clone() { + Some(id) if scrape_now => Some((id, endpoint.collector.fetch_stats().await)), + _ => None, + }; + Ok::<_, rocm_dash_core::traits::CollectorError>(( + discovered, + rows, + attributed_id, + stats, + )) + }) + .await; + for (endpoint, result) in results { + let (discovered, rows, attributed_id, stats) = match result { + Ok(result) => { + if endpoint.discover_external { + lemonade_discovery_warn = None; + } + result + } + Err(error) => { + // A failed external discovery still removes rows immediately, + // but reports why they disappeared. A never-seen endpoint + // with nothing listening stays quiet; one that answers with + // an HTTP or body error warns even on first contact. + // Managed failures retain only the existing hold. + if endpoint.discover_external { + let absent = endpoint.rows.is_empty() + && rocm_dash_collectors::lemonade::is_connect_error(&error); + lemonade_discovery_warn = + (!absent).then(|| endpoint.warning("health", &error)); + (Some(Vec::new()), Vec::new(), None, None) + } else { + scrape_warns.push(endpoint.warning("health", &error)); + continue; + } + } + }; + if let Some(services) = discovered { + Reconcile { + state: &mut runner.state, + telemetry: &mut telemetry, + tx: &tx, + persist: persist.as_ref(), + } + .reconcile_lemonade(&services, &mut discovery.lemonade_ids); + } + // Healthy contradictory/ambiguous evidence invalidates prior + // attribution immediately, even off the stats cadence. + for row in &rows { + if Some(&row.id) != attributed_id.as_ref() { + telemetry.gen_trackers.remove(&row.id); + } + } + if let Some((stats_id, stats)) = stats { + match stats { + Ok(sample) => { + if let Some(rate) = sample.gen_tps { + telemetry + .gen_trackers + .entry(stats_id) + .or_insert_with(|| { + GenerationObservationTracker::new(opts.instance_tick) + }) + .observe_direct(rate, cycle_at); + } + } + Err(error) => scrape_warns.push(endpoint.warning("stats", &error)), + } + } + } + } + + // Per-instance vLLM metric scrape, parallel, on its own cadence. + // Lemonade rows are scraped via their endpoint collectors above, so + // exclude them from the vLLM Prometheus targets. Managed non-vLLM services // are excluded too (their engine uses a different parser). if let Some(prom) = vllm.as_ref() && !runner.state.instances.is_empty() @@ -649,7 +912,7 @@ pub async fn run_loop( { let targets = vllm_scrape_targets( &runner.state.instances, - discovery.lemonade_id.as_deref(), + &discovery.lemonade_ids, &discovery.managed_non_vllm, ); let prom = prom.clone(); @@ -747,52 +1010,6 @@ pub async fn run_loop( } } - // Lemonade per-instance scrape — reports an instantaneous rate directly - // (`gen_tps`), so no counter-differencing. A scrape failure leaves the - // last-known fields and warns; it never panics. - if let (Some(l), Some(id)) = (lemonade.as_ref(), discovery.lemonade_id.clone()) - && tick_count.is_multiple_of(instance_ticks) - { - match l.fetch_stats().await { - Ok(sample) => { - // EAI-7960: drive the tracker with a direct rate reading when - // the sample carries one. A missing direct rate (None) does not - // erase or refresh the tracker; the held value stays valid until - // the validity window expires. - if let Some(rate) = sample.gen_tps { - telemetry - .gen_trackers - .entry(id.clone()) - .or_insert_with(|| { - GenerationObservationTracker::new(opts.instance_tick) - }) - .observe_direct(rate, cycle_at); - } - if let Some(mut inst) = runner.state.instances.get(&id).cloned() { - // gen_tps is written in the snapshot assembly below; - // set the KV/request fields immediately so they are - // present in state for subsequent snapshots. - inst.kv_cache_usage_pct = sample.kv_cache_usage_pct; - inst.running_reqs = sample.running_reqs; - inst.waiting_reqs = sample.waiting_reqs; - // See the vLLM scrape-success handler: a successful stats - // fetch proves the instance is serving; promote out of Starting. - if matches!(inst.status, InstanceStatus::Starting { .. }) { - inst.status = InstanceStatus::Ready; - } - runner.state.apply(StateEvent::InstanceUpserted(inst)); - } - } - Err(e) => { - trace!(id = %id, error = %e, "lemonade scrape failed"); - scrape_warns.push(format!("lemonade scrape: {e}")); - // EAI-7960: retain tracker during validity window on failure. - // gen_tps is assembled from the tracker snapshot below; - // no explicit clearing needed here. - } - } - } - // Per-process VRAM attribution: refresh the per-container map on the // instance cadence via one amd-smi `process` scrape, joining GPU-process // host PIDs to container ids through `/proc//cgroup`. On Err/empty @@ -841,6 +1058,10 @@ pub async fn run_loop( snapshot_latency(&mut telemetry.prev_ttft, &inst.container_id, cycle_at); (inst.tpot_ms, inst.tpot_observation) = snapshot_latency(&mut telemetry.prev_tpot, &inst.container_id, cycle_at); + inst.gen_tps_last_request = telemetry + .gen_trackers + .get(&inst.container_id) + .is_some_and(GenerationObservationTracker::is_direct); } // Derive per-instance efficiency now that this tick's GPU power is known // and gen_tps reflects the tracker state for this cycle. @@ -855,7 +1076,7 @@ pub async fn run_loop( for (inst, power_w) in instances.iter_mut().zip(attributed) { inst.power_w = power_w; inst.tokens_per_watt = - rocm_dash_core::efficiency::tokens_per_watt(inst.gen_tps, power_w); + rocm_dash_core::efficiency::instance_tokens_per_watt(inst, power_w); if inst.gen_tps.is_some() && !inst.gpu_ids.is_empty() && !gpus.is_empty() @@ -881,6 +1102,7 @@ pub async fn run_loop( entry.ttft_observation = inst.ttft_observation.clone(); entry.tpot_ms = inst.tpot_ms; entry.tpot_observation = inst.tpot_observation.clone(); + entry.gen_tps_last_request = inst.gen_tps_last_request; entry.tokens_per_watt = inst.tokens_per_watt; entry.power_w = inst.power_w; } @@ -907,6 +1129,7 @@ pub async fn run_loop( } warnings.extend(docker_warn.iter().cloned()); warnings.extend(registry_warn.iter().cloned()); + warnings.extend(lemonade_discovery_warn.iter().cloned()); warnings.extend(scrape_warns.iter().cloned()); let snap = Snapshot { timestamp: cycle_at, @@ -1018,6 +1241,7 @@ pub(crate) fn merge_discovery_refresh( inst.waiting_reqs = ex.waiting_reqs; inst.gen_tps = ex.gen_tps; inst.gen_tps_observation = ex.gen_tps_observation.clone(); + inst.gen_tps_last_request = ex.gen_tps_last_request; inst.tokens_per_watt = ex.tokens_per_watt; inst.power_w = ex.power_w; inst.ttft_ms = ex.ttft_ms; @@ -1028,347 +1252,6 @@ pub(crate) fn merge_discovery_refresh( inst } -/// Per-instance telemetry the loop carries between scrapes, keyed by container id. -#[derive(Default)] -struct TelemetryState { - /// Generation-throughput trackers. Constructed lazily from - /// `opts.instance_tick`; they survive scrape failures so the last-observed - /// rate is held for the validity window before clearing. - gen_trackers: HashMap, - /// Previous TTFT / TPOT histogram readings plus a freshness tracker per - /// instance, for the windowed avg-ms derivation (mirrors `gen_trackers`). - prev_ttft: LatencyWindows, - prev_tpot: LatencyWindows, -} - -impl TelemetryState { - /// Remove all per-service telemetry state for `id`: tracker, TTFT, and - /// TPOT baselines. Call on explicit removal (Docker Gone, Lemonade Gone, - /// managed Gone) and whenever an identity change requires a clean slate. - fn purge(&mut self, id: &str) { - self.gen_trackers.remove(id); - self.prev_ttft.remove(id); - self.prev_tpot.remove(id); - } - - /// Invalidate the generation tracker for `id` (reset baseline without - /// removing the map entry) and clear the TTFT/TPOT latency windows. - /// Called when an endpoint changes within the same container/service id so - /// the new scrape address starts with a fresh rate baseline. - fn invalidate_on_endpoint_change(&mut self, id: &str) { - if let Some(t) = self.gen_trackers.get_mut(id) { - t.invalidate(); - } - self.prev_ttft.remove(id); - self.prev_tpot.remove(id); - } -} - -/// What each discovery source currently has live, so a pass can tell a new -/// service from a known one and a known one from one that has gone. -#[derive(Default)] -struct DiscoveryState { - /// Container ids the last Docker pass saw. - known_instances: HashSet, - /// Ids the last managed-service registry pass surfaced. - known_services: HashSet, - /// The subset of the registry ids whose engine is NOT vLLM, excluded from - /// the vLLM Prometheus scrape so they aren't mis-parsed. - managed_non_vllm: HashSet, - /// Ancestry roots of the managed services the registry surfaced, used to - /// attribute GPU processes to a service when the cgroup join cannot (a - /// managed service is not a container). - managed_roots: Vec, - /// The id of the currently-live Lemonade instance, if any. - lemonade_id: Option, - /// How the registry reads have been going, so a condition that lasts many - /// passes is logged once rather than every discovery tick. - registry_health: RegistryHealth, -} - -impl DiscoveryState { - /// Record what a pass that read only part of the registry found. The services - /// it saw join the known set, so a later complete read can retire them, and - /// the known ones it did not see stay; the non-vLLM marks of the services it - /// saw are refreshed and the others kept, so a kept non-vLLM service is still - /// left out of the vLLM scrape. - fn keep_through_partial_registry(&mut self, disc: crate::registry::ManagedDiscovery) { - self.managed_non_vllm.retain(|id| !disc.seen.contains(id)); - self.managed_non_vllm.extend(disc.non_vllm); - // Roots of the services this pass saw are replaced; the unseen ones - // keep theirs, as their services stay known. - // A service this pass read and found stopped loses its roots too, or its - // PID would stay matchable for as long as an unrelated record is broken. - self.managed_roots - .retain(|r| !disc.seen.contains(&r.owner) && !disc.retired.contains(&r.owner)); - self.managed_roots.extend(disc.roots); - self.known_services.extend(disc.seen); - } -} - -/// What the managed-service registry's reads have reported so far. A failure -/// that lasts is logged on its first pass only, the recovery once, and a set of -/// malformed records each time it changes; the dashboard's warning badge, which -/// is cheap and does not scroll, carries the failure for as long as it lasts. -#[derive(Default)] -struct RegistryHealth { - /// The previous pass could not read the registry. - failing: bool, - /// The malformed records the last successful pass reported, so an unchanged - /// set is not reported again. - reported_malformed: Vec, -} - -/// What a discovery pass should say about the registry, apart from the services -/// it found. Kept as data so [`RegistryHealth::observe`] can be tested without a -/// log subscriber. -#[derive(Debug, Default, PartialEq, Eq)] -struct RegistryChange { - /// The first failed pass of a streak, with the error. - failed: Option, - /// The first successful pass after a streak of failures. - recovered: bool, - /// The malformed records, when the set differs from the one last reported - /// and is not empty. - malformed: Vec, -} - -impl RegistryHealth { - fn observe( - &mut self, - read: &Result, - ) -> RegistryChange { - let mut change = RegistryChange::default(); - match read { - Err(e) => { - if !self.failing { - self.failing = true; - change.failed = Some(e.to_string()); - } - } - Ok(read) => { - if self.failing { - self.failing = false; - change.recovered = true; - } - if read.malformed != self.reported_malformed { - self.reported_malformed.clone_from(&read.malformed); - change.malformed.clone_from(&read.malformed); - } - } - } - change - } -} - -impl RegistryChange { - fn log(&self) { - if let Some(error) = &self.failed { - warn!( - error = %error, - "managed-service registry could not be read; keeping the known services until it can be" - ); - } - if self.recovered { - info!("managed-service registry is readable again"); - } - if !self.malformed.is_empty() { - warn!( - files = ?self.malformed, - "skipping managed-service records that are not valid JSON" - ); - } - } -} - -/// The header warning for a pass that could not read the registry. -fn registry_read_warning(error: &crate::registry::RegistryReadError) -> String { - format!("managed registry: {error}; keeping the last known services") -} - -/// Which discovery source a service came from. It only selects the log wording -/// in [`Reconcile::announce`] and [`Reconcile::retire`]; those two steps behave -/// the same for every source. How a pass decides what is new, known or gone -/// differs: Docker and the registry go through `reconcile_set`, Lemonade through -/// `reconcile_lemonade`. -#[derive(Clone, Copy)] -enum DiscoverySource { - Docker, - Lemonade, - Managed, -} - -impl DiscoverySource { - fn log_discovered(self, svc: &DiscoveredService) { - match self { - Self::Docker => info!( - id = %svc.container_id, - name = %svc.container_name, - model = %svc.model_name, - "instance discovered" - ), - Self::Lemonade => info!( - id = %svc.container_id, - model = %svc.model_name, - "lemonade instance discovered" - ), - Self::Managed => info!(id = %svc.container_id, "managed service discovered"), - } - } - - fn log_gone(self, id: &str) { - match self { - Self::Docker => info!(id = %id, "instance gone"), - Self::Lemonade => info!(id = %id, "lemonade instance gone"), - Self::Managed => info!(id = %id, "managed service gone"), - } - } -} - -/// The loop state a discovery pass changes, and the stream it announces on. -struct Reconcile<'a> { - state: &'a mut State, - telemetry: &'a mut TelemetryState, - tx: &'a broadcast::Sender, - persist: Option<&'a Arc>>, -} - -impl Reconcile<'_> { - /// Upsert a (re)discovered service. Collector-owned fields carry over from - /// the instance already in state unless `carry_live` is false, which is how - /// a changed identity starts from a clean baseline. - fn upsert(&mut self, svc: &DiscoveredService, carry_live: bool) { - let existing = self.state.instances.get(&svc.container_id).cloned(); - let inst = merge_discovery_refresh(svc, if carry_live { existing.as_ref() } else { None }); - self.state.apply(StateEvent::InstanceUpserted(inst)); - } - - /// Announce a newly discovered service. InstanceDiscovered is a - /// discovery-metadata event: it carries a clean instance, with no prior - /// collector-owned telemetry. - fn announce(&self, source: DiscoverySource, svc: &DiscoveredService) { - source.log_discovered(svc); - broadcast_and_persist( - self.tx, - self.persist, - Event::InstanceDiscovered(instance_from_discovered(svc)), - ); - } - - /// Drop a service that has gone: purge its telemetry, remove it from state - /// and announce it. - fn retire(&mut self, source: DiscoverySource, id: &str) { - source.log_gone(id); - self.telemetry.purge(id); - self.state - .apply(StateEvent::InstanceRemoved(id.to_string())); - broadcast_and_persist( - self.tx, - self.persist, - Event::InstanceGone { - container_id: id.to_string(), - }, - ); - } - - /// Reconcile a source that reports its whole set each pass (Docker, the - /// managed-service registry). `known` is what the previous pass saw and - /// `seen` what this one saw; the caller stores `seen` as the next `known`. - /// - /// A service whose port changed under the same id has a new scrape address: - /// its rate baseline and latency windows are reset and no live fields carry - /// over. It is still the same service, so it is not announced again. - fn reconcile_set( - &mut self, - source: DiscoverySource, - svcs: &[DiscoveredService], - known: &HashSet, - seen: &HashSet, - ) { - self.upsert_set(source, svcs, known); - for gone in known.difference(seen) { - self.retire(source, gone); - } - } - - /// The half of [`Self::reconcile_set`] that adds and refreshes: upsert every - /// service in `svcs` and announce the ones `known` did not have. Retires - /// nothing, which is what a pass that could only see part of its source needs - /// (see [`DiscoveryState::keep_through_partial_registry`]). - fn upsert_set( - &mut self, - source: DiscoverySource, - svcs: &[DiscoveredService], - known: &HashSet, - ) { - for svc in svcs { - let same_endpoint = self - .state - .instances - .get(&svc.container_id) - .is_none_or(|ex| ex.port == svc.port); - if !same_endpoint { - self.telemetry - .invalidate_on_endpoint_change(&svc.container_id); - } - self.upsert(svc, same_endpoint); - if !known.contains(&svc.container_id) { - self.announce(source, svc); - } - } - } - - /// Reconcile the probe-based Lemonade source, which has at most one service. - /// - /// Its `container_id` encodes the host and port, so a port or host change - /// yields a new id. The old id is retired first (its telemetry is purged, so - /// a stale rate baseline, TTFT and TPOT do not bleed across; it leaves the - /// state and is announced gone), then the new id is announced, so the state - /// and every snapshot carry one Lemonade instance and the vLLM scrape never - /// polls the old endpoint. `None` means the endpoint did not answer, which - /// retires the current one. - fn reconcile_lemonade( - &mut self, - probed: Option, - current: &mut Option, - ) { - match probed { - Some(svc) => { - let is_new_id = current.as_deref() != Some(svc.container_id.as_str()); - if is_new_id && let Some(old_id) = current.take() { - self.retire(DiscoverySource::Lemonade, &old_id); - } - self.upsert(&svc, !is_new_id); - if is_new_id { - self.announce(DiscoverySource::Lemonade, &svc); - *current = Some(svc.container_id.clone()); - } - } - None => { - if let Some(id) = current.take() { - self.retire(DiscoverySource::Lemonade, &id); - } - } - } - } -} - -/// The `(id, port)` pairs the per-instance vLLM Prometheus scrape polls: every -/// instance with a port except the live Lemonade instance (its own collector -/// scrapes it) and the managed services whose engine is not vLLM. -fn vllm_scrape_targets( - instances: &HashMap, - lemonade_id: Option<&str>, - managed_non_vllm: &HashSet, -) -> Vec<(String, u16)> { - instances - .values() - .filter(|i| Some(i.container_id.as_str()) != lemonade_id) - .filter(|i| !managed_non_vllm.contains(&i.container_id)) - .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) - .collect() -} - /// Set `vram_used_mb`/`vram_total_mb` on each instance from the per-process /// attribution map, falling back to device-summed VRAM over the instance's /// GPUs when its container has no per-process entry. Pure — the runner does the @@ -1646,7 +1529,7 @@ mod tests { ); } - fn at(secs: i64) -> DateTime { + pub(super) fn at(secs: i64) -> DateTime { Utc.timestamp_opt(secs, 0).unwrap() } @@ -1654,7 +1537,7 @@ mod tests { const TICK: Duration = Duration::from_secs(2); - fn baseline(sum_s: f64, count: f64) -> LatencyWindow { + pub(super) fn baseline(sum_s: f64, count: f64) -> LatencyWindow { LatencyWindow { prev: Some((sum_s, count, at(10))), tracker: GenerationObservationTracker::new(TICK), @@ -1954,841 +1837,4 @@ mod tests { 120 ); } - - // -- Identity/endpoint cleanup helper tests ----------------------------------- - - fn make_tracker(tick_secs: u64) -> GenerationObservationTracker { - GenerationObservationTracker::new(Duration::from_secs(tick_secs)) - } - - /// `TelemetryState::purge` removes the tracker entry, TTFT baseline, and - /// TPOT baseline for the target id and leaves other ids untouched. - #[test] - fn telemetry_state_purge_removes_all_three_maps_for_id() { - let mut state = TelemetryState::default(); - - // Populate two ids. - state.gen_trackers.insert("a".into(), make_tracker(2)); - state.gen_trackers.insert("b".into(), make_tracker(2)); - state.prev_ttft.insert("a".into(), baseline(1.0, 1.0)); - state.prev_ttft.insert("b".into(), baseline(2.0, 2.0)); - state.prev_tpot.insert("a".into(), baseline(3.0, 3.0)); - state.prev_tpot.insert("b".into(), baseline(4.0, 4.0)); - - state.purge("a"); - - // "a" must be gone from all maps. - assert!( - !state.gen_trackers.contains_key("a"), - "tracker for a must be removed" - ); - assert!( - !state.prev_ttft.contains_key("a"), - "ttft for a must be removed" - ); - assert!( - !state.prev_tpot.contains_key("a"), - "tpot for a must be removed" - ); - // "b" must be unaffected. - assert!( - state.gen_trackers.contains_key("b"), - "tracker for b must survive" - ); - assert!(state.prev_ttft.contains_key("b"), "ttft for b must survive"); - assert!(state.prev_tpot.contains_key("b"), "tpot for b must survive"); - } - - /// `TelemetryState::invalidate_on_endpoint_change` invalidates the tracker - /// (resets its window so subsequent snapshots return None) and removes the - /// latency baselines, without removing the tracker entry itself. - #[test] - fn invalidate_on_endpoint_change_resets_tracker_and_clears_latency() { - let mut state = TelemetryState::default(); - - let mut tracker = make_tracker(2); - // Give the tracker a reading so it has a non-None baseline. - tracker.observe_counter(100.0, at(10)); - tracker.observe_counter(200.0, at(12)); - state.gen_trackers.insert("svc".into(), tracker); - state.prev_ttft.insert("svc".into(), baseline(1.0, 10.0)); - state.prev_tpot.insert("svc".into(), baseline(0.5, 8.0)); - - state.invalidate_on_endpoint_change("svc"); - - // Tracker still exists but its window is reset: snapshot returns None. - let (rate, _meta) = state - .gen_trackers - .get_mut("svc") - .expect("tracker entry must remain") - .snapshot(at(12)); - assert!(rate.is_none(), "invalidated tracker must yield None rate"); - // Latency baselines must be cleared. - assert!( - !state.prev_ttft.contains_key("svc"), - "ttft baseline must be cleared on endpoint change" - ); - assert!( - !state.prev_tpot.contains_key("svc"), - "tpot baseline must be cleared on endpoint change" - ); - } - - // ── Discovery reconcile ───────────────────────────────────────────────── - // - // The Docker source cannot be driven end to end without a container - // daemon, and `reconcile_set` is the code both it and the managed-service - // registry run, so these tests exercise it directly. The registry and - // Lemonade sources are also driven end to end in - // `tests/discovery_reconcile.rs`, except the registry's retirement of a - // vanished service and Lemonade's change of id, which only these tests - // cover. - - fn discovered(id: &str, port: u16) -> DiscoveredService { - DiscoveredService { - container_id: id.into(), - container_name: format!("name-{id}"), - model_name: "model".into(), - port: Some(port), - ..Default::default() - } - } - - fn id_set(ids: &[&str]) -> HashSet { - ids.iter().map(|s| (*s).to_string()).collect() - } - - /// The reducer state, telemetry and event stream a reconcile pass works on. - struct Pass { - state: State, - telemetry: TelemetryState, - tx: broadcast::Sender, - rx: broadcast::Receiver, - } - - impl Pass { - fn new() -> Self { - let (tx, rx) = broadcast::channel(64); - Self { - state: State::default(), - telemetry: TelemetryState::default(), - tx, - rx, - } - } - - fn reconcile(&mut self) -> Reconcile<'_> { - Reconcile { - state: &mut self.state, - telemetry: &mut self.telemetry, - tx: &self.tx, - persist: None, - } - } - - /// The announcements broadcast since the last call, as `discovered ` - /// and `gone ` strings. - fn announced(&mut self) -> Vec { - let mut out = Vec::new(); - while let Ok(ev) = self.rx.try_recv() { - match ev { - Event::InstanceDiscovered(i) => { - out.push(format!("discovered {}", i.container_id)); - } - Event::InstanceGone { container_id } => { - out.push(format!("gone {container_id}")); - } - _ => {} - } - } - out - } - - /// Give `id` a live KV reading and a latency baseline, as a scrape would. - fn scrape(&mut self, id: &str) { - let mut inst = self.state.instances.get(id).cloned().expect("instance"); - inst.kv_cache_usage_pct = Some(25.0); - self.state.apply(StateEvent::InstanceUpserted(inst)); - self.telemetry - .prev_ttft - .insert(id.into(), baseline(1.0, 10.0)); - self.telemetry - .prev_tpot - .insert(id.into(), baseline(0.5, 8.0)); - } - - fn kv(&self, id: &str) -> Option { - self.state - .instances - .get(id) - .and_then(|i| i.kv_cache_usage_pct) - } - } - - #[test] - fn reconcile_set_announces_a_new_service_once() { - let mut pass = Pass::new(); - let svcs = [discovered("a", 8000)]; - - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &svcs, - &id_set(&[]), - &id_set(&["a"]), - ); - assert_eq!(pass.announced(), ["discovered a"]); - assert!(pass.state.instances.contains_key("a")); - - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &svcs, - &id_set(&["a"]), - &id_set(&["a"]), - ); - assert!( - pass.announced().is_empty(), - "a known service must not be announced again" - ); - } - - #[test] - fn reconcile_set_keeps_live_fields_while_the_endpoint_is_unchanged() { - let mut pass = Pass::new(); - let svcs = [discovered("a", 8000)]; - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &svcs, - &id_set(&[]), - &id_set(&["a"]), - ); - pass.scrape("a"); - - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &svcs, - &id_set(&["a"]), - &id_set(&["a"]), - ); - - assert_eq!(pass.kv("a"), Some(25.0)); - assert!(pass.telemetry.prev_ttft.contains_key("a")); - assert!(pass.telemetry.prev_tpot.contains_key("a")); - } - - #[test] - fn reconcile_set_starts_clean_when_the_port_moves_under_the_same_id() { - let mut pass = Pass::new(); - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &[discovered("a", 8000)], - &id_set(&[]), - &id_set(&["a"]), - ); - pass.scrape("a"); - pass.announced(); - - pass.reconcile().reconcile_set( - DiscoverySource::Docker, - &[discovered("a", 9000)], - &id_set(&["a"]), - &id_set(&["a"]), - ); - - assert_eq!(pass.state.instances["a"].port, Some(9000)); - assert_eq!(pass.kv("a"), None, "the old endpoint's reading leaked"); - assert!(!pass.telemetry.prev_ttft.contains_key("a")); - assert!(!pass.telemetry.prev_tpot.contains_key("a")); - assert!( - pass.announced().is_empty(), - "a port change is not a new or a gone service" - ); - } - - #[test] - fn reconcile_set_retires_a_vanished_service_and_keeps_the_others() { - let mut pass = Pass::new(); - let svcs = [discovered("a", 8000), discovered("b", 8001)]; - pass.reconcile().reconcile_set( - DiscoverySource::Managed, - &svcs, - &id_set(&[]), - &id_set(&["a", "b"]), - ); - pass.scrape("a"); - pass.scrape("b"); - pass.announced(); - - pass.reconcile().reconcile_set( - DiscoverySource::Managed, - &svcs[..1], - &id_set(&["a", "b"]), - &id_set(&["a"]), - ); - - assert_eq!(pass.announced(), ["gone b"]); - assert!(!pass.state.instances.contains_key("b")); - assert!(!pass.telemetry.prev_ttft.contains_key("b")); - assert!(pass.state.instances.contains_key("a")); - assert!(pass.telemetry.prev_ttft.contains_key("a")); - } - - // ── Lemonade state matrix ─────────────────────────────────────────────── - // - // One Lemonade endpoint is probed per discovery pass. Its id encodes the host - // and port, so the states a probe can observe are: nothing ever answered; the - // first answer; the same endpoint again; the host or the port changed; the - // endpoint went quiet; it came back somewhere else; and the same answer - // arriving twice in one tick. Each row below checks the announcements, the - // instances a snapshot would carry, the telemetry kept per id, and the - // `(id, port)` pairs the vLLM scrape would poll, because a stale instance is - // visible on all four. - - fn lemonade_svc(host: &str, port: u16) -> DiscoveredService { - DiscoveredService { - container_id: format!("lemonade-{host}-{port}"), - container_name: "lemonade".into(), - model_name: "model".into(), - port: Some(port), - ..Default::default() - } - } - - impl Pass { - /// The ids a snapshot carries. The loop builds a snapshot's instances from - /// `state.instances`, so this is the dashboard's view of the pass. - fn instance_ids(&self) -> Vec { - let mut ids: Vec = self.state.instances.keys().cloned().collect(); - ids.sort(); - ids - } - - /// The ids the vLLM Prometheus scrape would poll while `current` is the - /// live Lemonade id. - fn scrape_targets(&self, current: Option<&str>) -> Vec { - let mut ids: Vec = - vllm_scrape_targets(&self.state.instances, current, &HashSet::new()) - .into_iter() - .map(|(id, _port)| id) - .collect(); - ids.sort(); - ids - } - - /// A live KV reading, latency baselines and a rate tracker for `id`. - fn give_live_telemetry(&mut self, id: &str) { - self.scrape(id); - self.telemetry - .gen_trackers - .insert(id.into(), make_tracker(2)); - } - - fn has_telemetry(&self, id: &str) -> bool { - self.telemetry.gen_trackers.contains_key(id) - || self.telemetry.prev_ttft.contains_key(id) - || self.telemetry.prev_tpot.contains_key(id) - } - } - - fn ids(ids: &[&str]) -> Vec { - ids.iter().map(|s| (*s).to_string()).collect() - } - - /// Discover `old`, give it live telemetry, then probe `new`. Returns the pass, - /// the loop's current Lemonade id and what the second probe announced. - fn lemonade_identity_change( - old: &DiscoveredService, - new: &DiscoveredService, - ) -> (Pass, Option, Vec) { - let mut pass = Pass::new(); - let mut current: Option = None; - pass.reconcile() - .reconcile_lemonade(Some(old.clone()), &mut current); - pass.give_live_telemetry(&old.container_id); - pass.announced(); - - pass.reconcile() - .reconcile_lemonade(Some(new.clone()), &mut current); - let announced = pass.announced(); - (pass, current, announced) - } - - #[test] - fn lemonade_that_never_answers_leaves_no_trace() { - let mut pass = Pass::new(); - let mut current: Option = None; - - pass.reconcile().reconcile_lemonade(None, &mut current); - pass.reconcile().reconcile_lemonade(None, &mut current); - - assert!(pass.announced().is_empty()); - assert_eq!(current, None); - assert!(pass.instance_ids().is_empty()); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn lemonade_first_answer_is_announced_once_and_left_to_its_own_collector() { - let mut pass = Pass::new(); - let mut current: Option = None; - - pass.reconcile() - .reconcile_lemonade(Some(lemonade_svc("127.0.0.1", 13305)), &mut current); - - assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); - assert_eq!(current.as_deref(), Some("lemonade-127.0.0.1-13305")); - assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); - assert!( - pass.scrape_targets(current.as_deref()).is_empty(), - "the live Lemonade instance is not a vLLM scrape target" - ); - } - - #[test] - fn lemonade_same_endpoint_again_keeps_its_instance_and_live_fields() { - let svc = lemonade_svc("127.0.0.1", 13305); - let id = svc.container_id.clone(); - let mut pass = Pass::new(); - let mut current: Option = None; - pass.reconcile() - .reconcile_lemonade(Some(svc.clone()), &mut current); - pass.give_live_telemetry(&id); - pass.announced(); - - pass.reconcile().reconcile_lemonade(Some(svc), &mut current); - - assert!(pass.announced().is_empty()); - assert_eq!(pass.instance_ids(), ids(&[&id])); - assert_eq!(pass.kv(&id), Some(25.0)); - assert!(pass.has_telemetry(&id)); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn lemonade_host_change_retires_the_old_instance_and_announces_the_new_one() { - let old = lemonade_svc("127.0.0.1", 13305); - let new = lemonade_svc("10.0.0.7", 13305); - let (pass, current, announced) = lemonade_identity_change(&old, &new); - - assert_eq!(current.as_deref(), Some(new.container_id.as_str())); - assert_eq!( - announced, - [ - "gone lemonade-127.0.0.1-13305", - "discovered lemonade-10.0.0.7-13305" - ] - ); - assert!(!pass.has_telemetry(&old.container_id)); - assert_eq!(pass.instance_ids(), ids(&["lemonade-10.0.0.7-13305"])); - assert!( - pass.scrape_targets(current.as_deref()).is_empty(), - "the old endpoint must not be polled as a vLLM target" - ); - } - - #[test] - fn lemonade_port_change_retires_the_old_instance_and_announces_the_new_one() { - let old = lemonade_svc("127.0.0.1", 13305); - let new = lemonade_svc("127.0.0.1", 13306); - let (pass, current, announced) = lemonade_identity_change(&old, &new); - - assert_eq!(current.as_deref(), Some(new.container_id.as_str())); - assert_eq!( - announced, - [ - "gone lemonade-127.0.0.1-13305", - "discovered lemonade-127.0.0.1-13306" - ] - ); - assert!(!pass.has_telemetry(&old.container_id)); - assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13306"])); - assert!( - pass.scrape_targets(current.as_deref()).is_empty(), - "the old endpoint must not be polled as a vLLM target" - ); - } - - #[test] - fn lemonade_endpoint_going_quiet_is_announced_gone_and_leaves_no_trace() { - let svc = lemonade_svc("127.0.0.1", 13305); - let id = svc.container_id.clone(); - let mut pass = Pass::new(); - let mut current: Option = None; - pass.reconcile().reconcile_lemonade(Some(svc), &mut current); - pass.give_live_telemetry(&id); - pass.announced(); - - pass.reconcile().reconcile_lemonade(None, &mut current); - - assert_eq!(pass.announced(), [format!("gone {id}")]); - assert_eq!(current, None); - assert!(pass.instance_ids().is_empty()); - assert!(!pass.has_telemetry(&id)); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn lemonade_coming_back_elsewhere_after_going_quiet_is_the_only_instance() { - let old = lemonade_svc("127.0.0.1", 13305); - let new = lemonade_svc("127.0.0.1", 13306); - let mut pass = Pass::new(); - let mut current: Option = None; - pass.reconcile() - .reconcile_lemonade(Some(old.clone()), &mut current); - pass.give_live_telemetry(&old.container_id); - pass.reconcile().reconcile_lemonade(None, &mut current); - assert_eq!( - pass.announced(), - [ - "discovered lemonade-127.0.0.1-13305", - "gone lemonade-127.0.0.1-13305" - ] - ); - - pass.reconcile() - .reconcile_lemonade(Some(new.clone()), &mut current); - - assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); - assert_eq!(current.as_deref(), Some(new.container_id.as_str())); - assert_eq!(pass.instance_ids(), ids(&[&new.container_id])); - assert!(!pass.has_telemetry(&old.container_id)); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn lemonade_answer_repeated_within_one_tick_is_announced_once() { - let svc = lemonade_svc("127.0.0.1", 13305); - let mut pass = Pass::new(); - let mut current: Option = None; - - pass.reconcile() - .reconcile_lemonade(Some(svc.clone()), &mut current); - pass.reconcile().reconcile_lemonade(Some(svc), &mut current); - - assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); - assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn lemonade_endpoint_moving_away_and_back_is_retired_each_time() { - let a = lemonade_svc("127.0.0.1", 13305); - let b = lemonade_svc("127.0.0.1", 13306); - let mut pass = Pass::new(); - let mut current: Option = None; - pass.reconcile() - .reconcile_lemonade(Some(a.clone()), &mut current); - pass.announced(); - - pass.reconcile().reconcile_lemonade(Some(b), &mut current); - assert_eq!( - pass.announced(), - [ - "gone lemonade-127.0.0.1-13305", - "discovered lemonade-127.0.0.1-13306" - ] - ); - pass.reconcile() - .reconcile_lemonade(Some(a.clone()), &mut current); - - assert_eq!( - pass.announced(), - [ - "gone lemonade-127.0.0.1-13306", - "discovered lemonade-127.0.0.1-13305" - ] - ); - assert_eq!(current.as_deref(), Some(a.container_id.as_str())); - assert_eq!(pass.instance_ids(), ids(&[&a.container_id])); - assert!(pass.scrape_targets(current.as_deref()).is_empty()); - } - - #[test] - fn reconcile_lemonade_follows_one_endpoint_through_its_lifecycle() { - let mut pass = Pass::new(); - let mut current: Option = None; - - pass.reconcile() - .reconcile_lemonade(Some(discovered("l1", 13305)), &mut current); - assert_eq!(pass.announced(), ["discovered l1"]); - assert_eq!(current.as_deref(), Some("l1")); - - pass.scrape("l1"); - pass.reconcile() - .reconcile_lemonade(Some(discovered("l1", 13305)), &mut current); - assert!( - pass.announced().is_empty(), - "same id must not be re-announced" - ); - assert_eq!(pass.kv("l1"), Some(25.0), "live fields carry over"); - - // A new id (the host or port changed) retires the old id, telemetry - // included, and announces the new one. - pass.reconcile() - .reconcile_lemonade(Some(discovered("l2", 13306)), &mut current); - assert_eq!(pass.announced(), ["gone l1", "discovered l2"]); - assert_eq!(current.as_deref(), Some("l2")); - assert!(!pass.telemetry.prev_ttft.contains_key("l1")); - // The superseded instance leaves the state (and so every snapshot), not - // only its telemetry. - assert!(!pass.state.instances.contains_key("l1")); - - pass.reconcile().reconcile_lemonade(None, &mut current); - assert_eq!(pass.announced(), ["gone l2"]); - assert_eq!(current, None); - assert!(!pass.state.instances.contains_key("l2")); - - pass.reconcile().reconcile_lemonade(None, &mut current); - assert!(pass.announced().is_empty(), "nothing left to retire"); - } - - // Registry read health: one test per row of the state matrix (first - // failure, repeated failure, recovery, empty, populated, malformed new, - // malformed unchanged, malformed cleared). - - fn registry_read( - malformed: &[&str], - ) -> Result { - Ok(crate::registry::RegistryRead { - records: Vec::new(), - malformed: malformed.iter().map(PathBuf::from).collect(), - }) - } - - fn registry_failure( - why: &str, - ) -> Result { - Err(crate::registry::RegistryReadError { - path: PathBuf::from("services"), - source: std::io::Error::other(why), - partial: crate::registry::RegistryRead::default(), - }) - } - - /// State matrix for what a partially read registry does to the discovery - /// state, one assertion group per row: a service it saw joins the known set - /// and its non-vLLM mark follows the record; one it did not see stays known - /// and keeps its mark; nothing leaves either set. - #[test] - fn partial_registry_read_adds_what_it_saw_and_forgets_nothing() { - let mut state = DiscoveryState { - known_services: id_set(&["old-vllm", "old-lemonade", "flips"]), - managed_non_vllm: id_set(&["old-lemonade", "flips"]), - ..DiscoveryState::default() - }; - - state.keep_through_partial_registry(crate::registry::ManagedDiscovery { - seen: id_set(&["new-lemonade", "new-vllm", "flips"]), - non_vllm: id_set(&["new-lemonade"]), - ..Default::default() - }); - - assert_eq!( - state.known_services, - id_set(&[ - "old-vllm", - "old-lemonade", - "flips", - "new-lemonade", - "new-vllm" - ]), - "the unseen services stay known and the new ones join them" - ); - assert_eq!( - state.managed_non_vllm, - id_set(&["old-lemonade", "new-lemonade"]), - "an unseen non-vLLM service keeps its mark, a seen one follows its record" - ); - } - - #[test] - fn partial_registry_read_refreshes_the_roots_it_saw_and_keeps_the_unseen() { - use rocm_dash_collectors::ancestry::Root; - let root = |pid, owner: &str| Root { - pid, - start_ticks: None, - owner: owner.into(), - }; - let mut state = DiscoveryState { - known_services: id_set(&["seen", "unseen"]), - managed_roots: vec![root(10, "seen"), root(20, "unseen")], - ..DiscoveryState::default() - }; - - // `seen` was relaunched under a new engine pid; `unseen` was not read. - state.keep_through_partial_registry(crate::registry::ManagedDiscovery { - seen: id_set(&["seen"]), - roots: vec![root(11, "seen")], - ..Default::default() - }); - - let mut got: Vec<_> = state - .managed_roots - .iter() - .map(|r| (r.pid, r.owner.as_str())) - .collect(); - got.sort_unstable(); - assert_eq!( - got, - vec![(11, "seen"), (20, "unseen")], - "a seen service's roots are replaced, an unseen one's survive" - ); - } - - #[test] - fn partial_registry_read_drops_the_roots_of_a_service_it_saw_stop() { - use rocm_dash_collectors::ancestry::Root; - let root = |pid, owner: &str| Root { - pid, - start_ticks: None, - owner: owner.into(), - }; - let mut state = DiscoveryState { - managed_roots: vec![root(10, "stopped"), root(20, "unseen")], - ..DiscoveryState::default() - }; - // `stopped` was read this pass and is no longer running; an unrelated - // record failed to read, which is what makes this a partial pass. - state.keep_through_partial_registry(crate::registry::ManagedDiscovery { - retired: id_set(&["stopped"]), - ..Default::default() - }); - let owners: Vec<_> = state - .managed_roots - .iter() - .map(|r| r.owner.as_str()) - .collect(); - assert_eq!( - owners, - vec!["unseen"], - "a stopped service's PID must not stay matchable" - ); - } - - #[test] - fn partial_registry_read_that_saw_nothing_keeps_every_root() { - use rocm_dash_collectors::ancestry::Root; - let mut state = DiscoveryState { - managed_roots: vec![Root { - pid: 10, - start_ticks: Some(5), - owner: "a".into(), - }], - ..DiscoveryState::default() - }; - state.keep_through_partial_registry(crate::registry::ManagedDiscovery::default()); - assert_eq!(state.managed_roots.len(), 1); - } - - #[test] - fn partial_registry_read_that_saw_nothing_changes_nothing() { - let mut state = DiscoveryState { - known_services: id_set(&["a", "b"]), - managed_non_vllm: id_set(&["b"]), - ..DiscoveryState::default() - }; - - state.keep_through_partial_registry(crate::registry::ManagedDiscovery::default()); - - assert_eq!(state.known_services, id_set(&["a", "b"])); - assert_eq!(state.managed_non_vllm, id_set(&["b"])); - } - - #[test] - fn registry_health_reports_the_first_failure_of_a_streak_once() { - let mut health = RegistryHealth::default(); - - let first = health.observe(®istry_failure("disk gone")); - assert_eq!( - first.failed.as_deref(), - Some("cannot read services: disk gone"), - "the first failed pass names the error" - ); - assert!(!first.recovered); - - for pass in 2..=5 { - assert_eq!( - health.observe(®istry_failure("disk gone")), - RegistryChange::default(), - "failed pass {pass} of the same streak must stay quiet" - ); - } - } - - #[test] - fn registry_health_reports_recovery_once_and_a_new_streak_again() { - let mut health = RegistryHealth::default(); - health.observe(®istry_failure("one")); - health.observe(®istry_failure("one")); - - let recovered = health.observe(®istry_read(&[])); - assert!(recovered.recovered); - assert!(recovered.failed.is_none()); - assert_eq!( - health.observe(®istry_read(&[])), - RegistryChange::default(), - "only the first good pass after a streak is a recovery" - ); - - let again = health.observe(®istry_failure("two")); - assert_eq!(again.failed.as_deref(), Some("cannot read services: two")); - } - - #[test] - fn registry_health_is_quiet_for_an_empty_or_populated_registry() { - let mut health = RegistryHealth::default(); - // A healthy read, empty or not, says nothing: there is no failure to - // report and no failure to have recovered from. - for _ in 0..3 { - assert_eq!( - health.observe(®istry_read(&[])), - RegistryChange::default() - ); - } - } - - #[test] - fn registry_health_reports_a_malformed_set_when_it_changes() { - let mut health = RegistryHealth::default(); - - let first = health.observe(®istry_read(&["a.json"])); - assert_eq!(first.malformed, [PathBuf::from("a.json")]); - assert_eq!( - health.observe(®istry_read(&["a.json"])), - RegistryChange::default(), - "an unchanged malformed set is not reported again" - ); - - let grown = health.observe(®istry_read(&["a.json", "b.json"])); - assert_eq!( - grown.malformed, - [PathBuf::from("a.json"), PathBuf::from("b.json")], - "a newly malformed record reports the whole set" - ); - - let cleared = health.observe(®istry_read(&[])); - assert!( - cleared.malformed.is_empty(), - "clearing the set is not a warning" - ); - let back = health.observe(®istry_read(&["a.json"])); - assert_eq!( - back.malformed, - [PathBuf::from("a.json")], - "a record that goes bad again after being fixed is reported again" - ); - } - - #[test] - fn registry_health_keeps_the_malformed_set_across_a_read_failure() { - let mut health = RegistryHealth::default(); - health.observe(®istry_read(&["a.json"])); - health.observe(®istry_failure("blip")); - - let after = health.observe(®istry_read(&["a.json"])); - assert!(after.recovered); - assert!( - after.malformed.is_empty(), - "the same corrupt record is not news just because a read failed in between" - ); - } } diff --git a/crates/rocm-dash-daemon/src/runner/discovery.rs b/crates/rocm-dash-daemon/src/runner/discovery.rs new file mode 100644 index 0000000000..3ac172bfcc --- /dev/null +++ b/crates/rocm-dash-daemon/src/runner/discovery.rs @@ -0,0 +1,1306 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What the runner's discovery passes remember between ticks, and how a pass +//! reconciles what a source reports against it: the per-instance telemetry the +//! loop carries ([`TelemetryState`]), what each source has live +//! ([`DiscoveryState`]), how the managed-service registry's reads have gone +//! ([`RegistryHealth`]), and the upsert/announce/retire steps every source shares +//! ([`Reconcile`]). Split out of `runner.rs` to keep it within the module-size +//! limit; `run_loop` drives all of it. + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use rocm_dash_core::metrics::Instance; +use rocm_dash_core::observation::GenerationObservationTracker; +use rocm_dash_core::protocol::Event; +use rocm_dash_core::state::{State, StateEvent}; +use rocm_dash_core::traits::DiscoveredService; +use tokio::sync::broadcast; +use tracing::{info, warn}; + +use super::{ + LatencyWindows, broadcast_and_persist, instance_from_discovered, merge_discovery_refresh, +}; +use crate::persist::SessionWriter; + +/// Per-instance telemetry the loop carries between scrapes, keyed by container id. +#[derive(Default)] +pub(super) struct TelemetryState { + /// Generation-throughput trackers. Constructed lazily from + /// `opts.instance_tick`; they survive scrape failures so the last-observed + /// rate is held for the validity window before clearing. + pub(super) gen_trackers: HashMap, + /// Previous TTFT / TPOT histogram readings plus a freshness tracker per + /// instance, for the windowed avg-ms derivation (mirrors `gen_trackers`). + pub(super) prev_ttft: LatencyWindows, + pub(super) prev_tpot: LatencyWindows, +} + +impl TelemetryState { + /// Remove all per-service telemetry state for `id`: tracker, TTFT, and + /// TPOT baselines. Call on explicit removal (Docker Gone, Lemonade Gone, + /// managed Gone) and whenever an identity change requires a clean slate. + pub(super) fn purge(&mut self, id: &str) { + self.gen_trackers.remove(id); + self.prev_ttft.remove(id); + self.prev_tpot.remove(id); + } + + /// Invalidate the generation tracker for `id` (reset baseline without + /// removing the map entry) and clear the TTFT/TPOT latency windows. + /// Called when an endpoint changes within the same container/service id so + /// the new scrape address starts with a fresh rate baseline. + pub(super) fn invalidate_on_endpoint_change(&mut self, id: &str) { + if let Some(t) = self.gen_trackers.get_mut(id) { + t.invalidate(); + } + self.prev_ttft.remove(id); + self.prev_tpot.remove(id); + } +} + +/// What each discovery source currently has live, so a pass can tell a new +/// service from a known one and a known one from one that has gone. +#[derive(Default)] +pub(super) struct DiscoveryState { + /// Container ids the last Docker pass saw. + pub(super) known_instances: HashSet, + /// Ids the last managed-service registry pass surfaced. + pub(super) known_services: HashSet, + /// The subset of the registry ids whose engine is NOT vLLM, excluded from + /// the vLLM Prometheus scrape so they aren't mis-parsed. + pub(super) managed_non_vllm: HashSet, + /// Model identities of the externally discovered Lemonade rows. + pub(super) lemonade_ids: HashSet, + /// Registry-managed Lemonade endpoints, scraped read-only for telemetry. + pub(super) managed_lemonade: Vec, + /// Ancestry roots of the managed services the registry surfaced, used to + /// attribute GPU processes to a service when the cgroup join cannot (a + /// managed service is not a container). + pub(super) managed_roots: Vec, + /// How the registry reads have been going, so a condition that lasts many + /// passes is logged once rather than every discovery tick. + pub(super) registry_health: RegistryHealth, +} + +impl DiscoveryState { + /// Fold a partial registry read into the managed Lemonade targets and return + /// the previous set: the targets this pass read are replaced by its own, the + /// ones it did not see are kept, and one it read and found stopped is + /// dropped (it is in `retired`, never in `seen`). Keeping a stopped target + /// would poll a dead port and, if anything later bound it, group that + /// service's row with the live one and lose the live one's throughput. + pub(super) fn keep_lemonade_through_partial_registry( + &mut self, + disc: &mut crate::registry::ManagedDiscovery, + ) -> Vec { + let mut next: Vec<_> = self + .managed_lemonade + .iter() + .filter(|target| { + !disc.seen.contains(&target.service_id) + && !disc.retired.contains(&target.service_id) + }) + .cloned() + .collect(); + next.append(&mut disc.lemonade); + std::mem::replace(&mut self.managed_lemonade, next) + } + + /// Record what a pass that read only part of the registry found. The services + /// it saw join the known set, so a later complete read can retire them, and + /// the known ones it did not see stay; the non-vLLM marks of the services it + /// saw are refreshed and the others kept, so a kept non-vLLM service is still + /// left out of the vLLM scrape. + pub(super) fn keep_through_partial_registry( + &mut self, + disc: crate::registry::ManagedDiscovery, + ) { + self.managed_non_vllm.retain(|id| !disc.seen.contains(id)); + self.managed_non_vllm.extend(disc.non_vllm); + // Roots of the services this pass saw are replaced; the unseen ones + // keep theirs, as their services stay known. + // A service this pass read and found stopped loses its roots too, or its + // PID would stay matchable for as long as an unrelated record is broken. + self.managed_roots + .retain(|r| !disc.seen.contains(&r.owner) && !disc.retired.contains(&r.owner)); + self.managed_roots.extend(disc.roots); + self.known_services.extend(disc.seen); + } +} + +/// What the managed-service registry's reads have reported so far. A failure +/// that lasts is logged on its first pass only, the recovery once, and a set of +/// malformed records each time it changes; the dashboard's warning badge, which +/// is cheap and does not scroll, carries the failure for as long as it lasts. +#[derive(Default)] +pub(super) struct RegistryHealth { + /// The previous pass could not read the registry. + pub(super) failing: bool, + /// The malformed records the last successful pass reported, so an unchanged + /// set is not reported again. + pub(super) reported_malformed: Vec, +} + +/// What a discovery pass should say about the registry, apart from the services +/// it found. Kept as data so [`RegistryHealth::observe`] can be tested without a +/// log subscriber. +#[derive(Debug, Default, PartialEq, Eq)] +pub(super) struct RegistryChange { + /// The first failed pass of a streak, with the error. + pub(super) failed: Option, + /// The first successful pass after a streak of failures. + pub(super) recovered: bool, + /// The malformed records, when the set differs from the one last reported + /// and is not empty. + pub(super) malformed: Vec, +} + +impl RegistryHealth { + pub(super) fn observe( + &mut self, + read: &Result, + ) -> RegistryChange { + let mut change = RegistryChange::default(); + match read { + Err(e) => { + if !self.failing { + self.failing = true; + change.failed = Some(e.to_string()); + } + } + Ok(read) => { + if self.failing { + self.failing = false; + change.recovered = true; + } + if read.malformed != self.reported_malformed { + self.reported_malformed.clone_from(&read.malformed); + change.malformed.clone_from(&read.malformed); + } + } + } + change + } +} + +impl RegistryChange { + pub(super) fn log(&self) { + if let Some(error) = &self.failed { + warn!( + error = %error, + "managed-service registry could not be read; keeping the known services until it can be" + ); + } + if self.recovered { + info!("managed-service registry is readable again"); + } + if !self.malformed.is_empty() { + warn!( + files = ?self.malformed, + "skipping managed-service records that are not valid JSON" + ); + } + } +} + +/// The header warning for a pass that could not read the registry. +pub(super) fn registry_read_warning(error: &crate::registry::RegistryReadError) -> String { + format!("managed registry: {error}; keeping the last known services") +} + +/// Which discovery source a service came from. It only selects the log wording +/// in [`Reconcile::announce`] and [`Reconcile::retire`]; those two steps behave +/// the same for every source. How a pass decides what is new, known or gone +/// differs: Docker and the registry go through `reconcile_set`, Lemonade through +/// `reconcile_lemonade`. +#[derive(Clone, Copy)] +pub(super) enum DiscoverySource { + Docker, + Lemonade, + Managed, +} + +impl DiscoverySource { + pub(super) fn log_discovered(self, svc: &DiscoveredService) { + match self { + Self::Docker => info!( + id = %svc.container_id, + name = %svc.container_name, + model = %svc.model_name, + "instance discovered" + ), + Self::Lemonade => info!( + id = %svc.container_id, + model = %svc.model_name, + "lemonade instance discovered" + ), + Self::Managed => info!(id = %svc.container_id, "managed service discovered"), + } + } + + pub(super) fn log_gone(self, id: &str) { + match self { + Self::Docker => info!(id = %id, "instance gone"), + Self::Lemonade => info!(id = %id, "lemonade instance gone"), + Self::Managed => info!(id = %id, "managed service gone"), + } + } +} + +/// The loop state a discovery pass changes, and the stream it announces on. +pub(super) struct Reconcile<'a> { + pub(super) state: &'a mut State, + pub(super) telemetry: &'a mut TelemetryState, + pub(super) tx: &'a broadcast::Sender, + pub(super) persist: Option<&'a Arc>>, +} + +impl Reconcile<'_> { + /// Upsert a (re)discovered service. Collector-owned fields carry over from + /// the instance already in state unless `carry_live` is false, which is how + /// a changed identity starts from a clean baseline. + pub(super) fn upsert(&mut self, svc: &DiscoveredService, carry_live: bool) { + let existing = self.state.instances.get(&svc.container_id).cloned(); + let inst = merge_discovery_refresh(svc, if carry_live { existing.as_ref() } else { None }); + self.state.apply(StateEvent::InstanceUpserted(inst)); + } + + /// Announce a newly discovered service. InstanceDiscovered is a + /// discovery-metadata event: it carries a clean instance, with no prior + /// collector-owned telemetry. + pub(super) fn announce(&self, source: DiscoverySource, svc: &DiscoveredService) { + source.log_discovered(svc); + broadcast_and_persist( + self.tx, + self.persist, + Event::InstanceDiscovered(instance_from_discovered(svc)), + ); + } + + /// Drop a service that has gone: purge its telemetry, remove it from state + /// and announce it. + pub(super) fn retire(&mut self, source: DiscoverySource, id: &str) { + source.log_gone(id); + self.telemetry.purge(id); + self.state + .apply(StateEvent::InstanceRemoved(id.to_string())); + broadcast_and_persist( + self.tx, + self.persist, + Event::InstanceGone { + container_id: id.to_string(), + }, + ); + } + + /// Reconcile a source that reports its whole set each pass (Docker, the + /// managed-service registry). `known` is what the previous pass saw and + /// `seen` what this one saw; the caller stores `seen` as the next `known`. + /// + /// A service whose port changed under the same id has a new scrape address: + /// its rate baseline and latency windows are reset and no live fields carry + /// over. It is still the same service, so it is not announced again. + pub(super) fn reconcile_set( + &mut self, + source: DiscoverySource, + svcs: &[DiscoveredService], + known: &HashSet, + seen: &HashSet, + ) { + self.upsert_set(source, svcs, known); + for gone in known.difference(seen) { + self.retire(source, gone); + } + } + + /// The half of [`Self::reconcile_set`] that adds and refreshes: upsert every + /// service in `svcs` and announce the ones `known` did not have. Retires + /// nothing, which is what a pass that could only see part of its source needs + /// (see [`DiscoveryState::keep_through_partial_registry`]). + pub(super) fn upsert_set( + &mut self, + source: DiscoverySource, + svcs: &[DiscoveredService], + known: &HashSet, + ) { + for svc in svcs { + let same_endpoint = self + .state + .instances + .get(&svc.container_id) + .is_none_or(|ex| ex.port == svc.port); + if !same_endpoint { + self.telemetry + .invalidate_on_endpoint_change(&svc.container_id); + } + self.upsert(svc, same_endpoint); + if !known.contains(&svc.container_id) { + self.announce(source, svc); + } + } + } + + /// Reconcile the probe-based Lemonade source: the models one endpoint + /// reports loaded. An empty set means the endpoint did not answer (or is + /// now registry-managed), which retires every current row. + /// + /// Each row's `container_id` encodes the host, the port and the model, so a + /// host or port change yields new ids. The ids that went are retired first + /// (their telemetry is purged, so a stale rate baseline, TTFT and TPOT do + /// not bleed across; they leave the state and are announced gone), then the + /// new ones are announced, so the state never carries a row for an endpoint + /// that moved and the vLLM scrape never polls the old one. `current` is + /// updated to the ids this pass saw. + pub(super) fn reconcile_lemonade( + &mut self, + services: &[DiscoveredService], + current: &mut HashSet, + ) { + let seen: HashSet = services + .iter() + .map(|svc| svc.container_id.clone()) + .collect(); + for gone in current.difference(&seen) { + self.retire(DiscoverySource::Lemonade, gone); + } + self.upsert_set(DiscoverySource::Lemonade, services, current); + *current = seen; + } + + /// Start a managed Lemonade service clean when its registry target moved to + /// another endpoint or model under the same service id: the rate baseline + /// and latency windows are reset and no live fields carry over, as for a + /// port change in [`Self::upsert_set`]. A target with no previous entry is + /// new and needs nothing reset. + pub(super) fn reset_moved_lemonade_targets( + &mut self, + svcs: &[DiscoveredService], + previous: &[crate::registry::ManagedLemonadeTarget], + current: &[crate::registry::ManagedLemonadeTarget], + ) { + for svc in svcs { + let id = &svc.container_id; + let Some(old) = previous.iter().find(|target| &target.service_id == id) else { + continue; + }; + let unchanged = current + .iter() + .find(|target| &target.service_id == id) + .is_some_and(|new| { + crate::registry::endpoints_match(&old.host, old.port, &new.host, new.port) + && old.model_ref == new.model_ref + && old.canonical_model_id == new.canonical_model_id + }); + if !unchanged && self.state.instances.contains_key(id) { + self.telemetry.invalidate_on_endpoint_change(id); + self.upsert(svc, false); + } + } + } +} + +/// The `(id, port)` pairs the per-instance vLLM Prometheus scrape polls: every +/// instance with a port except the discovered Lemonade rows (their endpoint +/// collector scrapes them) and the managed services whose engine is not vLLM. +pub(super) fn vllm_scrape_targets( + instances: &HashMap, + lemonade_ids: &HashSet, + managed_non_vllm: &HashSet, +) -> Vec<(String, u16)> { + instances + .values() + .filter(|i| !lemonade_ids.contains(&i.container_id)) + .filter(|i| !managed_non_vllm.contains(&i.container_id)) + .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::runner::tests::{at, baseline}; + use std::time::Duration; + + // -- Identity/endpoint cleanup helper tests ----------------------------------- + + fn target(id: &str, port: u16) -> crate::registry::ManagedLemonadeTarget { + crate::registry::ManagedLemonadeTarget { + service_id: id.into(), + host: "127.0.0.1".into(), + port, + model_ref: "m".into(), + canonical_model_id: "m".into(), + } + } + + /// A partial registry read replaces the targets it saw, keeps the ones it + /// did not see, and drops one it read and found stopped. + #[test] + fn a_partial_registry_read_keeps_unseen_lemonade_targets_and_drops_stopped_ones() { + let mut state = DiscoveryState { + managed_lemonade: vec![target("seen", 1), target("unseen", 2), target("stopped", 3)], + ..DiscoveryState::default() + }; + let mut disc = crate::registry::ManagedDiscovery { + svcs: Vec::new(), + seen: HashSet::from(["seen".to_string()]), + non_vllm: HashSet::new(), + lemonade: vec![target("seen", 10)], + past_attempts: 0, + roots: Vec::new(), + retired: HashSet::from(["stopped".to_string()]), + }; + let previous = state.keep_lemonade_through_partial_registry(&mut disc); + assert_eq!(previous.len(), 3, "the previous set is handed back"); + let mut now: Vec<_> = state + .managed_lemonade + .iter() + .map(|t| (t.service_id.as_str(), t.port)) + .collect(); + now.sort_unstable(); + assert_eq!(now, [("seen", 10), ("unseen", 2)]); + } + + fn make_tracker(tick_secs: u64) -> GenerationObservationTracker { + GenerationObservationTracker::new(Duration::from_secs(tick_secs)) + } + + /// `TelemetryState::purge` removes the tracker entry, TTFT baseline, and + /// TPOT baseline for the target id and leaves other ids untouched. + #[test] + fn telemetry_state_purge_removes_all_three_maps_for_id() { + let mut state = TelemetryState::default(); + + // Populate two ids. + state.gen_trackers.insert("a".into(), make_tracker(2)); + state.gen_trackers.insert("b".into(), make_tracker(2)); + state.prev_ttft.insert("a".into(), baseline(1.0, 1.0)); + state.prev_ttft.insert("b".into(), baseline(2.0, 2.0)); + state.prev_tpot.insert("a".into(), baseline(3.0, 3.0)); + state.prev_tpot.insert("b".into(), baseline(4.0, 4.0)); + + state.purge("a"); + + // "a" must be gone from all maps. + assert!( + !state.gen_trackers.contains_key("a"), + "tracker for a must be removed" + ); + assert!( + !state.prev_ttft.contains_key("a"), + "ttft for a must be removed" + ); + assert!( + !state.prev_tpot.contains_key("a"), + "tpot for a must be removed" + ); + // "b" must be unaffected. + assert!( + state.gen_trackers.contains_key("b"), + "tracker for b must survive" + ); + assert!(state.prev_ttft.contains_key("b"), "ttft for b must survive"); + assert!(state.prev_tpot.contains_key("b"), "tpot for b must survive"); + } + + /// `TelemetryState::invalidate_on_endpoint_change` invalidates the tracker + /// (resets its window so subsequent snapshots return None) and removes the + /// latency baselines, without removing the tracker entry itself. + #[test] + fn invalidate_on_endpoint_change_resets_tracker_and_clears_latency() { + let mut state = TelemetryState::default(); + + let mut tracker = make_tracker(2); + // Give the tracker a reading so it has a non-None baseline. + tracker.observe_counter(100.0, at(10)); + tracker.observe_counter(200.0, at(12)); + state.gen_trackers.insert("svc".into(), tracker); + state.prev_ttft.insert("svc".into(), baseline(1.0, 10.0)); + state.prev_tpot.insert("svc".into(), baseline(0.5, 8.0)); + + state.invalidate_on_endpoint_change("svc"); + + // Tracker still exists but its window is reset: snapshot returns None. + let (rate, _meta) = state + .gen_trackers + .get_mut("svc") + .expect("tracker entry must remain") + .snapshot(at(12)); + assert!(rate.is_none(), "invalidated tracker must yield None rate"); + // Latency baselines must be cleared. + assert!( + !state.prev_ttft.contains_key("svc"), + "ttft baseline must be cleared on endpoint change" + ); + assert!( + !state.prev_tpot.contains_key("svc"), + "tpot baseline must be cleared on endpoint change" + ); + } + + // ── Discovery reconcile ───────────────────────────────────────────────── + // + // The Docker source cannot be driven end to end without a container + // daemon, and `reconcile_set` is the code both it and the managed-service + // registry run, so these tests exercise it directly. The registry and + // Lemonade sources are also driven end to end in + // `tests/discovery_reconcile.rs`, except the registry's retirement of a + // vanished service and Lemonade's change of id, which only these tests + // cover. + + fn discovered(id: &str, port: u16) -> DiscoveredService { + DiscoveredService { + container_id: id.into(), + container_name: format!("name-{id}"), + model_name: "model".into(), + port: Some(port), + ..Default::default() + } + } + + fn id_set(ids: &[&str]) -> HashSet { + ids.iter().map(|s| (*s).to_string()).collect() + } + + /// The reducer state, telemetry and event stream a reconcile pass works on. + struct Pass { + state: State, + telemetry: TelemetryState, + tx: broadcast::Sender, + rx: broadcast::Receiver, + } + + impl Pass { + fn new() -> Self { + let (tx, rx) = broadcast::channel(64); + Self { + state: State::default(), + telemetry: TelemetryState::default(), + tx, + rx, + } + } + + fn reconcile(&mut self) -> Reconcile<'_> { + Reconcile { + state: &mut self.state, + telemetry: &mut self.telemetry, + tx: &self.tx, + persist: None, + } + } + + /// The announcements broadcast since the last call, as `discovered ` + /// and `gone ` strings. + fn announced(&mut self) -> Vec { + let mut out = Vec::new(); + while let Ok(ev) = self.rx.try_recv() { + match ev { + Event::InstanceDiscovered(i) => { + out.push(format!("discovered {}", i.container_id)); + } + Event::InstanceGone { container_id } => { + out.push(format!("gone {container_id}")); + } + _ => {} + } + } + out + } + + /// Give `id` a live KV reading and a latency baseline, as a scrape would. + fn scrape(&mut self, id: &str) { + let mut inst = self.state.instances.get(id).cloned().expect("instance"); + inst.kv_cache_usage_pct = Some(25.0); + self.state.apply(StateEvent::InstanceUpserted(inst)); + self.telemetry + .prev_ttft + .insert(id.into(), baseline(1.0, 10.0)); + self.telemetry + .prev_tpot + .insert(id.into(), baseline(0.5, 8.0)); + } + + fn kv(&self, id: &str) -> Option { + self.state + .instances + .get(id) + .and_then(|i| i.kv_cache_usage_pct) + } + } + + #[test] + fn reconcile_set_announces_a_new_service_once() { + let mut pass = Pass::new(); + let svcs = [discovered("a", 8000)]; + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&[]), + &id_set(&["a"]), + ); + assert_eq!(pass.announced(), ["discovered a"]); + assert!(pass.state.instances.contains_key("a")); + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&["a"]), + &id_set(&["a"]), + ); + assert!( + pass.announced().is_empty(), + "a known service must not be announced again" + ); + } + + #[test] + fn reconcile_set_keeps_live_fields_while_the_endpoint_is_unchanged() { + let mut pass = Pass::new(); + let svcs = [discovered("a", 8000)]; + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&[]), + &id_set(&["a"]), + ); + pass.scrape("a"); + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&["a"]), + &id_set(&["a"]), + ); + + assert_eq!(pass.kv("a"), Some(25.0)); + assert!(pass.telemetry.prev_ttft.contains_key("a")); + assert!(pass.telemetry.prev_tpot.contains_key("a")); + } + + #[test] + fn reconcile_set_starts_clean_when_the_port_moves_under_the_same_id() { + let mut pass = Pass::new(); + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &[discovered("a", 8000)], + &id_set(&[]), + &id_set(&["a"]), + ); + pass.scrape("a"); + pass.announced(); + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &[discovered("a", 9000)], + &id_set(&["a"]), + &id_set(&["a"]), + ); + + assert_eq!(pass.state.instances["a"].port, Some(9000)); + assert_eq!(pass.kv("a"), None, "the old endpoint's reading leaked"); + assert!(!pass.telemetry.prev_ttft.contains_key("a")); + assert!(!pass.telemetry.prev_tpot.contains_key("a")); + assert!( + pass.announced().is_empty(), + "a port change is not a new or a gone service" + ); + } + + #[test] + fn reconcile_set_retires_a_vanished_service_and_keeps_the_others() { + let mut pass = Pass::new(); + let svcs = [discovered("a", 8000), discovered("b", 8001)]; + pass.reconcile().reconcile_set( + DiscoverySource::Managed, + &svcs, + &id_set(&[]), + &id_set(&["a", "b"]), + ); + pass.scrape("a"); + pass.scrape("b"); + pass.announced(); + + pass.reconcile().reconcile_set( + DiscoverySource::Managed, + &svcs[..1], + &id_set(&["a", "b"]), + &id_set(&["a"]), + ); + + assert_eq!(pass.announced(), ["gone b"]); + assert!(!pass.state.instances.contains_key("b")); + assert!(!pass.telemetry.prev_ttft.contains_key("b")); + assert!(pass.state.instances.contains_key("a")); + assert!(pass.telemetry.prev_ttft.contains_key("a")); + } + + // ── Lemonade state matrix ─────────────────────────────────────────────── + // + // One Lemonade endpoint is probed per discovery pass. Its id encodes the host + // and port, so the states a probe can observe are: nothing ever answered; the + // first answer; the same endpoint again; the host or the port changed; the + // endpoint went quiet; it came back somewhere else; and the same answer + // arriving twice in one tick. Each row below checks the announcements, the + // instances a snapshot would carry, the telemetry kept per id, and the + // `(id, port)` pairs the vLLM scrape would poll, because a stale instance is + // visible on all four. + + fn lemonade_svc(host: &str, port: u16) -> DiscoveredService { + DiscoveredService { + container_id: format!("lemonade-{host}-{port}"), + container_name: "lemonade".into(), + model_name: "model".into(), + port: Some(port), + ..Default::default() + } + } + + impl Pass { + /// The ids a snapshot carries. The loop builds a snapshot's instances from + /// `state.instances`, so this is the dashboard's view of the pass. + fn instance_ids(&self) -> Vec { + let mut ids: Vec = self.state.instances.keys().cloned().collect(); + ids.sort(); + ids + } + + /// The ids the vLLM Prometheus scrape would poll while `current` holds + /// the live Lemonade ids. + fn scrape_targets(&self, current: &HashSet) -> Vec { + let mut ids: Vec = + vllm_scrape_targets(&self.state.instances, current, &HashSet::new()) + .into_iter() + .map(|(id, _port)| id) + .collect(); + ids.sort(); + ids + } + + /// A live KV reading, latency baselines and a rate tracker for `id`. + fn give_live_telemetry(&mut self, id: &str) { + self.scrape(id); + self.telemetry + .gen_trackers + .insert(id.into(), make_tracker(2)); + } + + fn has_telemetry(&self, id: &str) -> bool { + self.telemetry.gen_trackers.contains_key(id) + || self.telemetry.prev_ttft.contains_key(id) + || self.telemetry.prev_tpot.contains_key(id) + } + } + + fn ids(ids: &[&str]) -> Vec { + ids.iter().map(|s| (*s).to_string()).collect() + } + + /// Discover `old`, give it live telemetry, then probe `new`. Returns the pass, + /// the loop's current Lemonade ids and what the second probe announced. + fn lemonade_identity_change( + old: &DiscoveredService, + new: &DiscoveredService, + ) -> (Pass, HashSet, Vec) { + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(old), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.announced(); + + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(new), &mut current); + let announced = pass.announced(); + (pass, current, announced) + } + + #[test] + fn lemonade_that_never_answers_leaves_no_trace() { + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + + pass.reconcile().reconcile_lemonade(&[], &mut current); + pass.reconcile().reconcile_lemonade(&[], &mut current); + + assert!(pass.announced().is_empty()); + assert!(current.is_empty()); + assert!(pass.instance_ids().is_empty()); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn lemonade_first_answer_is_announced_once_and_left_to_its_own_collector() { + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + + pass.reconcile() + .reconcile_lemonade(&[lemonade_svc("127.0.0.1", 13305)], &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!( + current, + HashSet::from(["lemonade-127.0.0.1-13305".to_string()]) + ); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!( + pass.scrape_targets(¤t).is_empty(), + "the live Lemonade instance is not a vLLM scrape target" + ); + } + + #[test] + fn lemonade_same_endpoint_again_keeps_its_instance_and_live_fields() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&svc), &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(&[svc], &mut current); + + assert!(pass.announced().is_empty()); + assert_eq!(pass.instance_ids(), ids(&[&id])); + assert_eq!(pass.kv(&id), Some(25.0)); + assert!(pass.has_telemetry(&id)); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn lemonade_host_change_retires_the_old_instance_and_announces_the_new_one() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("10.0.0.7", 13305); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current, HashSet::from([new.container_id])); + assert_eq!( + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-10.0.0.7-13305" + ] + ); + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-10.0.0.7-13305"])); + assert!( + pass.scrape_targets(¤t).is_empty(), + "the old endpoint must not be polled as a vLLM target" + ); + } + + #[test] + fn lemonade_port_change_retires_the_old_instance_and_announces_the_new_one() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current, HashSet::from([new.container_id])); + assert_eq!( + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] + ); + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13306"])); + assert!( + pass.scrape_targets(¤t).is_empty(), + "the old endpoint must not be polled as a vLLM target" + ); + } + + #[test] + fn lemonade_endpoint_going_quiet_is_announced_gone_and_leaves_no_trace() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + pass.reconcile().reconcile_lemonade(&[svc], &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(&[], &mut current); + + assert_eq!(pass.announced(), [format!("gone {id}")]); + assert!(current.is_empty()); + assert!(pass.instance_ids().is_empty()); + assert!(!pass.has_telemetry(&id)); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn lemonade_coming_back_elsewhere_after_going_quiet_is_the_only_instance() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&old), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.reconcile().reconcile_lemonade(&[], &mut current); + assert_eq!( + pass.announced(), + [ + "discovered lemonade-127.0.0.1-13305", + "gone lemonade-127.0.0.1-13305" + ] + ); + + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&new), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); + assert_eq!(current, HashSet::from([new.container_id.clone()])); + assert_eq!(pass.instance_ids(), ids(&[&new.container_id])); + assert!(!pass.has_telemetry(&old.container_id)); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn lemonade_answer_repeated_within_one_tick_is_announced_once() { + let svc = lemonade_svc("127.0.0.1", 13305); + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&svc), &mut current); + pass.reconcile().reconcile_lemonade(&[svc], &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn lemonade_endpoint_moving_away_and_back_is_retired_each_time() { + let a = lemonade_svc("127.0.0.1", 13305); + let b = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&a), &mut current); + pass.announced(); + + pass.reconcile().reconcile_lemonade(&[b], &mut current); + assert_eq!( + pass.announced(), + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] + ); + pass.reconcile() + .reconcile_lemonade(std::slice::from_ref(&a), &mut current); + + assert_eq!( + pass.announced(), + [ + "gone lemonade-127.0.0.1-13306", + "discovered lemonade-127.0.0.1-13305" + ] + ); + assert_eq!(current, HashSet::from([a.container_id.clone()])); + assert_eq!(pass.instance_ids(), ids(&[&a.container_id])); + assert!(pass.scrape_targets(¤t).is_empty()); + } + + #[test] + fn reconcile_lemonade_follows_one_endpoint_through_its_lifecycle() { + let mut pass = Pass::new(); + let mut current: HashSet = HashSet::new(); + + pass.reconcile() + .reconcile_lemonade(&[discovered("l1", 13305)], &mut current); + assert_eq!(pass.announced(), ["discovered l1"]); + assert_eq!(current, HashSet::from(["l1".to_string()])); + + pass.scrape("l1"); + pass.reconcile() + .reconcile_lemonade(&[discovered("l1", 13305)], &mut current); + assert!( + pass.announced().is_empty(), + "same id must not be re-announced" + ); + assert_eq!(pass.kv("l1"), Some(25.0), "live fields carry over"); + + // A new id (the host or port changed) retires the old id, telemetry + // included, and announces the new one. + pass.reconcile() + .reconcile_lemonade(&[discovered("l2", 13306)], &mut current); + assert_eq!(pass.announced(), ["gone l1", "discovered l2"]); + assert_eq!(current, HashSet::from(["l2".to_string()])); + assert!(!pass.telemetry.prev_ttft.contains_key("l1")); + // The superseded instance leaves the state (and so every snapshot), not + // only its telemetry. + assert!(!pass.state.instances.contains_key("l1")); + + pass.reconcile().reconcile_lemonade(&[], &mut current); + assert_eq!(pass.announced(), ["gone l2"]); + assert!(current.is_empty()); + assert!(!pass.state.instances.contains_key("l2")); + + pass.reconcile().reconcile_lemonade(&[], &mut current); + assert!(pass.announced().is_empty(), "nothing left to retire"); + } + + // Registry read health: one test per row of the state matrix (first + // failure, repeated failure, recovery, empty, populated, malformed new, + // malformed unchanged, malformed cleared). + + fn registry_read( + malformed: &[&str], + ) -> Result { + Ok(crate::registry::RegistryRead { + records: Vec::new(), + malformed: malformed.iter().map(PathBuf::from).collect(), + }) + } + + fn registry_failure( + why: &str, + ) -> Result { + Err(crate::registry::RegistryReadError { + path: PathBuf::from("services"), + source: std::io::Error::other(why), + partial: crate::registry::RegistryRead::default(), + }) + } + + /// State matrix for what a partially read registry does to the discovery + /// state, one assertion group per row: a service it saw joins the known set + /// and its non-vLLM mark follows the record; one it did not see stays known + /// and keeps its mark; nothing leaves either set. + #[test] + fn partial_registry_read_adds_what_it_saw_and_forgets_nothing() { + let mut state = DiscoveryState { + known_services: id_set(&["old-vllm", "old-lemonade", "flips"]), + managed_non_vllm: id_set(&["old-lemonade", "flips"]), + ..DiscoveryState::default() + }; + + state.keep_through_partial_registry(crate::registry::ManagedDiscovery { + seen: id_set(&["new-lemonade", "new-vllm", "flips"]), + non_vllm: id_set(&["new-lemonade"]), + ..Default::default() + }); + + assert_eq!( + state.known_services, + id_set(&[ + "old-vllm", + "old-lemonade", + "flips", + "new-lemonade", + "new-vllm" + ]), + "the unseen services stay known and the new ones join them" + ); + assert_eq!( + state.managed_non_vllm, + id_set(&["old-lemonade", "new-lemonade"]), + "an unseen non-vLLM service keeps its mark, a seen one follows its record" + ); + } + + #[test] + fn partial_registry_read_that_saw_nothing_changes_nothing() { + let mut state = DiscoveryState { + known_services: id_set(&["a", "b"]), + managed_non_vllm: id_set(&["b"]), + ..DiscoveryState::default() + }; + + state.keep_through_partial_registry(crate::registry::ManagedDiscovery::default()); + + assert_eq!(state.known_services, id_set(&["a", "b"])); + assert_eq!(state.managed_non_vllm, id_set(&["b"])); + } + + #[test] + fn registry_health_reports_the_first_failure_of_a_streak_once() { + let mut health = RegistryHealth::default(); + + let first = health.observe(®istry_failure("disk gone")); + assert_eq!( + first.failed.as_deref(), + Some("cannot read services: disk gone"), + "the first failed pass names the error" + ); + assert!(!first.recovered); + + for pass in 2..=5 { + assert_eq!( + health.observe(®istry_failure("disk gone")), + RegistryChange::default(), + "failed pass {pass} of the same streak must stay quiet" + ); + } + } + + #[test] + fn registry_health_reports_recovery_once_and_a_new_streak_again() { + let mut health = RegistryHealth::default(); + health.observe(®istry_failure("one")); + health.observe(®istry_failure("one")); + + let recovered = health.observe(®istry_read(&[])); + assert!(recovered.recovered); + assert!(recovered.failed.is_none()); + assert_eq!( + health.observe(®istry_read(&[])), + RegistryChange::default(), + "only the first good pass after a streak is a recovery" + ); + + let again = health.observe(®istry_failure("two")); + assert_eq!(again.failed.as_deref(), Some("cannot read services: two")); + } + + #[test] + fn registry_health_is_quiet_for_an_empty_or_populated_registry() { + let mut health = RegistryHealth::default(); + // A healthy read, empty or not, says nothing: there is no failure to + // report and no failure to have recovered from. + for _ in 0..3 { + assert_eq!( + health.observe(®istry_read(&[])), + RegistryChange::default() + ); + } + } + + #[test] + fn registry_health_reports_a_malformed_set_when_it_changes() { + let mut health = RegistryHealth::default(); + + let first = health.observe(®istry_read(&["a.json"])); + assert_eq!(first.malformed, [PathBuf::from("a.json")]); + assert_eq!( + health.observe(®istry_read(&["a.json"])), + RegistryChange::default(), + "an unchanged malformed set is not reported again" + ); + + let grown = health.observe(®istry_read(&["a.json", "b.json"])); + assert_eq!( + grown.malformed, + [PathBuf::from("a.json"), PathBuf::from("b.json")], + "a newly malformed record reports the whole set" + ); + + let cleared = health.observe(®istry_read(&[])); + assert!( + cleared.malformed.is_empty(), + "clearing the set is not a warning" + ); + let back = health.observe(®istry_read(&["a.json"])); + assert_eq!( + back.malformed, + [PathBuf::from("a.json")], + "a record that goes bad again after being fixed is reported again" + ); + } + + #[test] + fn registry_health_keeps_the_malformed_set_across_a_read_failure() { + let mut health = RegistryHealth::default(); + health.observe(®istry_read(&["a.json"])); + health.observe(®istry_failure("blip")); + + let after = health.observe(®istry_read(&["a.json"])); + assert!(after.recovered); + assert!( + after.malformed.is_empty(), + "the same corrupt record is not news just because a read failed in between" + ); + } + + #[test] + fn partial_registry_read_refreshes_the_roots_it_saw_and_keeps_the_unseen() { + use rocm_dash_collectors::ancestry::Root; + let root = |pid, owner: &str| Root { + pid, + start_ticks: None, + owner: owner.into(), + }; + let mut state = DiscoveryState { + known_services: id_set(&["seen", "unseen"]), + managed_roots: vec![root(10, "seen"), root(20, "unseen")], + ..DiscoveryState::default() + }; + + // `seen` was relaunched under a new engine pid; `unseen` was not read. + state.keep_through_partial_registry(crate::registry::ManagedDiscovery { + seen: id_set(&["seen"]), + roots: vec![root(11, "seen")], + ..Default::default() + }); + + let mut got: Vec<_> = state + .managed_roots + .iter() + .map(|r| (r.pid, r.owner.as_str())) + .collect(); + got.sort_unstable(); + assert_eq!( + got, + vec![(11, "seen"), (20, "unseen")], + "a seen service's roots are replaced, an unseen one's survive" + ); + } + + #[test] + fn partial_registry_read_drops_the_roots_of_a_service_it_saw_stop() { + use rocm_dash_collectors::ancestry::Root; + let root = |pid, owner: &str| Root { + pid, + start_ticks: None, + owner: owner.into(), + }; + let mut state = DiscoveryState { + managed_roots: vec![root(10, "stopped"), root(20, "unseen")], + ..DiscoveryState::default() + }; + // `stopped` was read this pass and is no longer running; an unrelated + // record failed to read, which is what makes this a partial pass. + state.keep_through_partial_registry(crate::registry::ManagedDiscovery { + retired: id_set(&["stopped"]), + ..Default::default() + }); + let owners: Vec<_> = state + .managed_roots + .iter() + .map(|r| r.owner.as_str()) + .collect(); + assert_eq!( + owners, + vec!["unseen"], + "a stopped service's PID must not stay matchable" + ); + } + + #[test] + fn partial_registry_read_that_saw_nothing_keeps_every_root() { + use rocm_dash_collectors::ancestry::Root; + let mut state = DiscoveryState { + managed_roots: vec![Root { + pid: 10, + start_ticks: Some(5), + owner: "a".into(), + }], + ..DiscoveryState::default() + }; + state.keep_through_partial_registry(crate::registry::ManagedDiscovery::default()); + assert_eq!(state.managed_roots.len(), 1); + } +} diff --git a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs index 570c0a656c..76d16e0dd2 100644 --- a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs +++ b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs @@ -964,13 +964,66 @@ fn lemonade_responder(path: &str) -> String { } } +fn two_model_responder(path: &str) -> String { + match path { + "/api/v1/health" => http( + 200, + "application/json", + r#"{"status":"ok","model_loaded":"Mock-Lemonade-1B","all_models_loaded":[{"model_name":"Mock-Lemonade-1B"},{"model_name":"Mock-Lemonade-2B"}]}"#, + ), + "/api/v1/stats" => http(200, "application/json", r#"{"tokens_per_second":33.5}"#), + _ => http(404, "text/plain", ""), + } +} + +/// An endpoint with two models loaded is two rows: one id per model, each +/// spelled out here (endpoint, then the model's UTF-8 bytes in hex) so that +/// collapsing them into one row per endpoint fails this test. +#[tokio::test] +async fn each_loaded_lemonade_model_is_announced_as_its_own_row() { + let server = spawn_server(two_model_responder).await; + let ids = [ + format!( + "lemonade-127.0.0.1-{}-model-4d6f636b2d4c656d6f6e6164652d3142", + server.port + ), + format!( + "lemonade-127.0.0.1-{}-model-4d6f636b2d4c656d6f6e6164652d3242", + server.port + ), + ]; + let (mut rx, _runner) = spawn_runner(lemonade_opts(server.port)); + let mut announced = Vec::new(); + while announced.len() < 2 { + let inst = next_event(&mut rx, |ev| match ev { + Event::InstanceDiscovered(inst) => Some(inst.clone()), + _ => None, + }) + .await; + announced.push((inst.container_id, inst.model_name)); + } + announced.sort(); + assert_eq!( + announced, + [ + (ids[0].clone(), "Mock-Lemonade-1B".to_string()), + (ids[1].clone(), "Mock-Lemonade-2B".to_string()), + ] + ); +} + /// A reachable Lemonade endpoint is announced once with its health-reported /// model, scraped for its rate, and announced gone when the endpoint stops /// answering, after which it leaves the snapshots. #[tokio::test] async fn lemonade_endpoint_is_announced_scraped_and_announced_gone() { let server = spawn_server(lemonade_responder).await; - let id = format!("lemonade-127.0.0.1-{}", server.port); + // The identity is spelled out, not derived from the production function, + // so a change to it fails here: endpoint, then the model's UTF-8 bytes in hex. + let id = format!( + "lemonade-127.0.0.1-{}-model-4d6f636b2d4c656d6f6e6164652d3142", + server.port + ); let (mut rx, _runner) = spawn_runner(lemonade_opts(server.port)); let announced = next_event(&mut rx, |ev| match ev { diff --git a/crates/rocm-dash-daemon/tests/telemetry_contract.rs b/crates/rocm-dash-daemon/tests/telemetry_contract.rs index aae7b5a41c..5ac26a3b66 100644 --- a/crates/rocm-dash-daemon/tests/telemetry_contract.rs +++ b/crates/rocm-dash-daemon/tests/telemetry_contract.rs @@ -27,15 +27,24 @@ //! Scenarios 1–3, 5–8 verify already-correct or distinguishably-observable //! behaviour. All scenarios now serve as regression guards. //! +//! The `lemonade_*` tests below the matrix cover Lemonade discovery and +//! ownership at the same seam, and are not rows in it: identity and the shared +//! last-request rate across several loaded models; external discovery, its +//! warnings (a seen endpoint closing, one never seen, an HTTP error on first +//! contact, recovery) and its removal of rows; a managed record taking over, or +//! being removed from, an endpoint; managed rewrites that reset the attributed +//! rate; scrape-only ticks; and IPv4/IPv6 dual-stack binds. The test names say +//! which. +//! //! Timing: `tick_override = 200 ms`, `discovery_tick = 200 ms`, //! `instance_tick = 400 ms` (scrape every 2nd base tick). //! Expiry window boundary: `clamp(3 × 400ms, 6 s, 30 s) = 6 s`. //! -//! ## Frozen observation schema (proposed — do not implement in production during this task) +//! ## Observation schema (as shipped in `rocm-dash-core`) //! -//! The controller-reviewed additive schema for EAI-7960 observation metadata. -//! Production types (`rocm-dash-core`) are unchanged; this block is the -//! candidate contract for the implementer to reference. +//! The additive schema for EAI-7960 observation metadata. The production types +//! are `ObservationMetadata` and `ObservationFreshness` in `rocm-dash-core`'s +//! `metrics.rs`; this block is the contract the tests below assert against. //! //! ```rust,ignore //! // Existing value carrier — unchanged, remains the sole numeric source. @@ -102,6 +111,1381 @@ use rocm_dash_daemon::bench_ring::BenchRing; use rocm_dash_daemon::runner::{self, RunnerOptions}; use rocm_dash_daemon::snapshot_ring::SnapshotRing; +/// Read-only Lemonade fixture with a mutable health body. The fixture records +/// unexpected requests, including any attempt to manage or stop the server. +async fn lemonade_fixture( + health: Arc>, + unexpected: Arc, +) -> (u16, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let stats = Arc::new(Mutex::new(r#"{"tokens_per_second":42.0}"#.to_string())); + lemonade_fixture_on(listener, health, stats, unexpected).await +} + +/// Same fixture on a caller-chosen listener with a mutable stats body. +// Async so call-sites read `.await` like `lemonade_fixture`. +#[allow(clippy::unused_async)] +async fn lemonade_fixture_on( + listener: TcpListener, + health: Arc>, + stats: Arc>, + unexpected: Arc, +) -> (u16, tokio::task::JoinHandle<()>) { + let port = listener.local_addr().unwrap().port(); + let task = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let health = health.clone(); + let stats = stats.clone(); + let unexpected = unexpected.clone(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + let body = if request.starts_with("GET /api/v1/health ") { + health.lock().unwrap().clone() + } else if request.starts_with("GET /api/v1/stats ") { + stats.lock().unwrap().clone() + } else { + unexpected.fetch_add(1, Ordering::Relaxed); + "{}".to_string() + }; + let response = if body == "HTTP_503" { + HTTP_503.to_string() + } else { + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + }; + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + (port, task) +} + +/// With the shipped cadence (discovery more often than instance scrapes) a +/// discovery-only tick asks the endpoint for its health but must not fetch its +/// stats: those ride the instance cadence. Here discovery runs every tick and the +/// instance scrape every fifth, so stats requests stay well under the health ones. +#[tokio::test] +async fn lemonade_stats_are_fetched_on_the_instance_cadence_not_every_discovery_tick() { + let health_hits = Arc::new(AtomicU64::new(0)); + let stats_hits = Arc::new(AtomicU64::new(0)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = { + let (health_hits, stats_hits) = (health_hits.clone(), stats_hits.clone()); + tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let (health_hits, stats_hits) = (health_hits.clone(), stats_hits.clone()); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + let body = if request.starts_with("GET /api/v1/health ") { + health_hits.fetch_add(1, Ordering::Relaxed); + r#"{"status":"ok","model_loaded":"Model-A","all_models_loaded":[{"model_name":"Model-A"}]}"# + } else { + stats_hits.fetch_add(1, Ordering::Relaxed); + r#"{"tokens_per_second":42.0}"# + }; + let response = format!( + "HTTP/1.1 200 OK +Content-Length: {} +Connection: close + +{body}", + body.len() + ); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }) + }; + let (tx, _rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK * 5, + ..RunnerOptions::default() + }, + ) + .await; + let deadline = tokio::time::Instant::now() + SCENARIO_DEADLINE; + while health_hits.load(Ordering::Relaxed) < 15 { + assert!( + tokio::time::Instant::now() < deadline, + "the endpoint was never polled 15 times" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + task.abort(); + server.abort(); + let (health, stats) = ( + health_hits.load(Ordering::Relaxed), + stats_hits.load(Ordering::Relaxed), + ); + assert!(stats >= 1, "the instance cadence still fetches stats"); + assert!( + stats * 3 <= health, + "stats must follow the instance cadence (1 in 5), not every poll: {stats} stats for {health} health requests" + ); +} + +#[tokio::test] +async fn lemonade_multiple_models_keep_identity_and_do_not_share_last_request_rate() { + let health = Arc::new(Mutex::new(r#"{"status":"ok","all_models_loaded":[{"model_name":"Model-A"},{"model_name":"Model-B"},{"model_name":"Model-A"}]}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health.clone(), unexpected.clone()).await; + let (tx, mut rx) = broadcast::channel(64); + let clock_dir = tempfile::tempdir().unwrap(); + let clock = clock_dir.path().join("clock"); + std::fs::write(&clock, "hold").unwrap(); + let opts = RunnerOptions { + test_clock_offset_path: Some(clock), + enable_lemonade: true, + lemonade_port: port, + disable_vllm_metrics: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }; + let task = spawn_runner(tx, opts).await; + let result = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2).then(|| snap.instances.clone()) + }) + .await; + if result.is_err() { + task.abort(); + server.abort(); + } + let rows = result.expect("two deduplicated loaded model rows"); + assert!(rows.iter().all(|row| row.gen_tps.is_none())); + let model_a_id = rows + .iter() + .find(|row| row.model_name == "Model-A") + .unwrap() + .container_id + .clone(); + let model_b_id = rows + .iter() + .find(|row| row.model_name == "Model-B") + .unwrap() + .container_id + .clone(); + assert_ne!(model_a_id, model_b_id); + *health.lock().unwrap() = r#"{"status":"ok","model_loaded":"Model-B","all_models_loaded":[{"model_name":"Model-B"},{"model_name":"Model-A"}]}"#.to_string(); + wait_for_snapshot(&mut rx, |snap| { + let a = snap + .instances + .iter() + .find(|row| row.container_id == model_a_id)?; + let b = snap + .instances + .iter() + .find(|row| row.container_id == model_b_id)?; + (a.gen_tps.is_none() && b.gen_tps.is_none()).then_some(()) + }) + .await + .expect("last access is not last request; multiple loaded models have no attributable rate"); + *health.lock().unwrap() = + r#"{"status":"ok","all_models_loaded":[{"model_name":"Model-A"}]}"#.to_string(); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 + && snap.instances[0].container_id == model_a_id + && snap.instances[0].gen_tps == Some(42.0)) + .then_some(()) + }) + .await + .expect("unloaded model disappears; singleton receives rate"); + // Freeze observation time: expiry cannot make a missing invalidation pass, + // even if the test is descheduled or waits through its entire deadline. + drain_snapshots(&mut rx); + *health.lock().unwrap() = + r#"{"status":"ok","model_loaded":"Model-A","all_models_loaded":["Model-A","Model-B"]}"# + .to_string(); + let ambiguous = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2).then(|| snap.instances.clone()) + }) + .await + .expect("adding another model produces both rows on the next discovery"); + assert!( + ambiguous + .iter() + .all(|row| row.gen_tps.is_none() && row.gen_tps_observation.is_none()), + "the first multi-model snapshot must invalidate the singleton rate and observation" + ); + *health.lock().unwrap() = + r#"{"status":"error","all_models_loaded":["Model-A","Model-B"]}"#.to_string(); + wait_for_snapshot(&mut rx, |snap| snap.instances.is_empty().then_some(())) + .await + .expect("non-ok health never remains ready"); + *health.lock().unwrap() = r#"{"status":"ok","all_models_loaded":["Model-A"]}"#.to_string(); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].container_id == model_a_id).then_some(()) + }) + .await + .expect("healthy recovery restores the same model identity"); + *health.lock().unwrap() = r#"{"status":"ok","all_models_loaded":[]}"#.to_string(); + wait_for_snapshot(&mut rx, |snap| snap.instances.is_empty().then_some(())) + .await + .expect("healthy router without loaded models is not serving"); + task.abort(); + server.abort(); + assert_eq!( + unexpected.load(Ordering::Relaxed), + 0, + "only read-only health/stats requests are allowed" + ); +} + +#[tokio::test] +async fn lemonade_managed_nondefault_port_scrapes_even_when_external_discovery_disabled() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("managed.json"), format!(r#"{{"service_id":"managed-lemonade","engine":"lemonade","model_ref":"model-alias","canonical_model_id":"Model-A","host":"127.0.0.1","port":{port},"status":"ready"}}"#)).unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let opts = RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }; + let task = spawn_runner(tx, opts).await; + let result = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 + && snap.instances[0].container_id == "managed-lemonade" + && snap.instances[0].managed + && snap.instances[0].gen_tps == Some(42.0) + && snap.instances[0].gen_tps_last_request) + .then_some(()) + }) + .await; + task.abort(); + server.abort(); + result.expect("managed Lemonade is scraped at its registry port without external discovery"); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +/// A successful Lemonade scrape proves the server answers health and stats, but +/// the row's status still comes from the registry record: unlike a Docker +/// container, whose first good scrape is the only sign it is serving, a managed +/// service's own record says when it is ready. The telemetry reaching the row in +/// the same snapshot is what shows the scrape did succeed. +#[tokio::test] +async fn a_successful_lemonade_scrape_leaves_a_starting_managed_row_starting() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A","all_models_loaded":[{"model_name":"Model-A"}]}"# + .to_string(), + )); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("managed.json"), format!(r#"{{"service_id":"starting-lemonade","engine":"lemonade","model_ref":"Model-A","canonical_model_id":"Model-A","host":"127.0.0.1","port":{port},"status":"starting"}}"#)).unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let snapshot = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].gen_tps == Some(42.0)).then(|| snap.clone()) + }) + .await; + task.abort(); + server.abort(); + let snapshot = snapshot.expect("the scrape reaches the managed row"); + let row = &snapshot.instances[0]; + assert_eq!(row.container_id, "starting-lemonade"); + assert!( + matches!( + row.status, + rocm_dash_core::metrics::InstanceStatus::Starting { .. } + ), + "a successful scrape must not promote the registry's Starting: {:?}", + row.status + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_equivalent_endpoint_records_do_not_duplicate_singleton_rate() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + for (id, host) in [("first", "127.0.0.1"), ("second", "LOCALHOST")] { + std::fs::write(dir.path().join(format!("{id}.json")), format!(r#"{{"service_id":"{id}","engine":"lemonade","model_ref":"Model-A","host":"{host}","port":{port},"status":"ready"}}"#)).unwrap(); + } + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let result = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2).then(|| snap.instances.clone()) + }) + .await; + task.abort(); + server.abort(); + let rows = result.expect("both registry identities remain present"); + assert!( + rows.iter() + .all(|row| row.managed && row.gen_tps.is_none() && row.gen_tps_observation.is_none()), + "equivalent endpoint records cannot each receive its singleton rate: {rows:?}" + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_external_endpoint_deduplicates_managed_owner_then_reappears_after_record_removal() +{ + let health = Arc::new(Mutex::new(r#"{"status":"ok","all_models_loaded":[{"model_name":"Model-A"},{"model_name":"Model-B"}]}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + let record = dir.path().join("managed.json"); + std::fs::write(&record, format!(r#"{{"service_id":"managed-lemonade","engine":"lemonade","model_ref":"Model-A","host":"localhost","port":{port},"status":"ready"}}"#)).unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let opts = RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }; + let task = spawn_runner(tx, opts).await; + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 + && snap.instances[0].container_id == "managed-lemonade" + && snap.instances[0].gen_tps.is_none()) + .then_some(()) + }) + .await + .expect("managed endpoint is not duplicated or assigned ambiguous stats"); + std::fs::remove_file(record).unwrap(); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2 + && snap.instances.iter().all(|row| { + !row.managed && row.container_id != "managed-lemonade" && row.gen_tps.is_none() + })) + .then_some(()) + }) + .await + .expect("removing the registry file does not stop an externally reachable server"); + task.abort(); + server.abort(); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_managed_record_taking_over_endpoint_drops_existing_external_rows() { + let health = Arc::new(Mutex::new(r#"{"status":"ok","all_models_loaded":[{"model_name":"Model-A"},{"model_name":"Model-B"}]}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let opts = RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }; + let task = spawn_runner(tx, opts).await; + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2 && snap.instances.iter().all(|row| !row.managed)).then_some(()) + }) + .await + .expect("external rows are discovered before any managed record exists"); + std::fs::write(dir.path().join("managed.json"), format!(r#"{{"service_id":"managed-lemonade","engine":"lemonade","model_ref":"Model-A","host":"localhost","port":{port},"status":"ready"}}"#)).unwrap(); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 + && snap.instances[0].container_id == "managed-lemonade" + && snap.instances[0].managed) + .then_some(()) + }) + .await + .expect("managed takeover removes the external rows and keeps the managed row"); + task.abort(); + server.abort(); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_external_health_failure_warns_and_removes_rows_without_grace() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health.clone(), unexpected.clone()).await; + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let row = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].gen_tps == Some(42.0)) + .then(|| snap.instances[0].clone()) + }) + .await + .expect("external singleton established"); + assert!( + row.gen_tps_last_request, + "a Lemonade rate is the last request's and must be flagged as such" + ); + drain_snapshots(&mut rx); + *health.lock().unwrap() = "HTTP_503".to_string(); + // Discovery runs every tick here, so the rows go on the next one. A hold of + // even a few ticks (the observation validity window is 6 s, some thirty + // ticks) is a regression: count the snapshots that still carry the row. + let still_listed = AtomicU64::new(0); + let result = wait_for_snapshot(&mut rx, |snap| { + if snap.instances.is_empty() { + Some(snap.warnings.clone()) + } else { + still_listed.fetch_add(1, Ordering::Relaxed); + None + } + }) + .await; + task.abort(); + server.abort(); + let warnings = result.expect("external failure removes rows on next discovery, without grace"); + assert!( + still_listed.load(Ordering::Relaxed) <= 3, + "rows were held for {} snapshots after the health failure", + still_listed.load(Ordering::Relaxed) + ); + assert!( + warnings + .iter() + .any(|warning| warning.contains("lemonade health") + && warning.contains(&port.to_string()) + && warning.contains("503")), + "failed disappearance carries the health error: {warnings:?}" + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_seen_external_endpoint_closing_warns_and_removes_rows() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + wait_for_snapshot(&mut rx, |snap| (snap.instances.len() == 1).then_some(())) + .await + .unwrap(); + server.abort(); + let _ = server.await; + drain_snapshots(&mut rx); + let result = wait_for_snapshot(&mut rx, |snap| { + snap.instances.is_empty().then(|| snap.warnings.clone()) + }) + .await; + let warnings = result.expect("closing a seen endpoint removes its rows"); + assert!( + warnings + .iter() + .any(|warning| warning.contains("lemonade health") + && warning.contains(&port.to_string())), + "disappearance must name the endpoint: {warnings:?}" + ); + + // The endpoint comes back on the same port: its row returns and the warning + // about its disappearance is cleared, not kept for as long as the daemon runs. + let listener = TcpListener::bind(("127.0.0.1", port)).await.unwrap(); + let stats = Arc::new(Mutex::new(r#"{"tokens_per_second":42.0}"#.to_string())); + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let (_, revived) = lemonade_fixture_on(listener, health, stats, unexpected.clone()).await; + drain_snapshots(&mut rx); + let recovered = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1).then(|| snap.warnings.clone()) + }) + .await; + task.abort(); + revived.abort(); + let warnings = recovered.expect("a recovered endpoint's row returns"); + assert!( + !warnings.iter().any(|warning| warning.contains("lemonade")), + "recovery must clear the disappearance warning: {warnings:?}" + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn lemonade_never_seen_absent_external_endpoint_is_quiet() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let snap = wait_for_snapshot(&mut rx, |snap| Some(snap.clone())).await; + task.abort(); + let snap = snap.expect("absent endpoint does not block snapshots"); + assert!(snap.instances.is_empty()); + assert!( + !snap + .warnings + .iter() + .any(|warning| warning.contains("lemonade")), + "normal absent external endpoint is not a noisy warning" + ); +} + +/// A never-seen external endpoint that answers with an HTTP error is present +/// but unusable (auth needed or broken), so it warns on first contact. +async fn never_seen_endpoint_answering_warns(status_line: &'static str, needle: &str) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let _ = socket.read(&mut bytes).await; + let response = format!( + "HTTP/1.1 {status_line}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let warnings = wait_for_snapshot(&mut rx, |snap| { + snap.warnings + .iter() + .any(|w| w.contains("lemonade")) + .then(|| snap.warnings.clone()) + }) + .await; + task.abort(); + server.abort(); + let warnings = warnings.expect("first-contact HTTP error warns"); + assert!( + warnings.iter().any(|w| w.contains("lemonade health") + && w.contains(&port.to_string()) + && w.contains(needle)), + "warning names the endpoint and status: {warnings:?}" + ); +} + +#[tokio::test] +async fn lemonade_never_seen_endpoint_answering_401_warns() { + never_seen_endpoint_answering_warns("401 Unauthorized", "401").await; +} + +#[tokio::test] +async fn lemonade_never_seen_endpoint_answering_503_warns() { + never_seen_endpoint_answering_warns("503 Service Unavailable", "503").await; +} + +#[tokio::test] +async fn lemonade_protected_managed_endpoint_keeps_registry_status_without_telemetry_warning() { + for status in ["401 Unauthorized", "403 Forbidden"] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let unexpected = Arc::new(AtomicU64::new(0)); + let seen = unexpected.clone(); + let server = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let seen = seen.clone(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + if !request.starts_with("GET /api/v1/health ") + || request.to_ascii_lowercase().contains("authorization:") + { + seen.fetch_add(1, Ordering::Relaxed); + } + let _ = socket.write_all(format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").as_bytes()).await; + }); + } + }); + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("managed.json"), format!(r#"{{"service_id":"protected","engine":"lemonade","model_ref":"Model-A","host":"127.0.0.1","port":{port},"status":"starting"}}"#)).unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let snapshot = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1).then(|| snap.clone()) + }) + .await; + task.abort(); + server.abort(); + let snapshot = snapshot.unwrap(); + let row = &snapshot.instances[0]; + assert!(row.managed); + assert_eq!(row.container_id, "protected"); + assert!(matches!( + row.status, + rocm_dash_core::metrics::InstanceStatus::Starting { .. } + )); + assert!(row.gen_tps.is_none() && row.gen_tps_observation.is_none()); + assert!( + !snapshot + .warnings + .iter() + .any(|warning| warning.contains("lemonade")), + "protected telemetry is unavailable, not a recurring health failure: {:?}", + snapshot.warnings + ); + assert_eq!( + unexpected.load(Ordering::Relaxed), + 0, + "no stats, fallback, credentials or lifecycle requests" + ); + } +} + +/// A slow health response can finish only after the other endpoint's stats +/// request. Both serial orders fail: fast health waits for slow health to start. +#[tokio::test] +async fn lemonade_independent_endpoints_overlap_and_health_failures_warn() { + let slow_started = Arc::new(tokio::sync::Notify::new()); + let fast_stats = Arc::new(tokio::sync::Notify::new()); + let fast_stats_count = Arc::new(AtomicU64::new(0)); + let failed = Arc::new(AtomicBool::new(false)); + let unexpected = Arc::new(AtomicU64::new(0)); + let mut servers = Vec::new(); + let dir = tempfile::tempdir().unwrap(); + for (id, model, slow) in [("slow", "Model-S", true), ("fast", "Model-F", false)] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + std::fs::write(dir.path().join(format!("{id}.json")), format!(r#"{{"service_id":"{id}","engine":"lemonade","model_ref":"{model}","host":"127.0.0.1","port":{port},"status":"ready"}}"#)).unwrap(); + let slow_started = slow_started.clone(); + let fast_stats = fast_stats.clone(); + let fast_stats_count = fast_stats_count.clone(); + let failed = failed.clone(); + let unexpected = unexpected.clone(); + let health_succeeded = Arc::new(AtomicBool::new(false)); + servers.push(tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let slow_started = slow_started.clone(); + let fast_stats = fast_stats.clone(); + let fast_stats_count = fast_stats_count.clone(); + let failed = failed.clone(); + let unexpected = unexpected.clone(); + let health_succeeded = health_succeeded.clone(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + let response = if request.starts_with("GET /api/v1/health ") { + health_succeeded.store(false, Ordering::SeqCst); + if slow && failed.load(Ordering::SeqCst) { + HTTP_503.to_string() + } else { + let overlap = if slow { + slow_started.notify_one(); + timeout(Duration::from_secs(1), fast_stats.notified()).await + } else { + timeout(Duration::from_secs(1), slow_started.notified()).await + }; + if overlap.is_ok() { + health_succeeded.store(true, Ordering::SeqCst); + http_ok(&format!(r#"{{"status":"ok","model_loaded":"{model}"}}"#)) + } else { + HTTP_503.to_string() + } + } + } else if request.starts_with("GET /api/v1/stats ") { + if !health_succeeded.swap(false, Ordering::SeqCst) { + unexpected.fetch_add(1, Ordering::Relaxed); + } + if !slow { + fast_stats_count.fetch_add(1, Ordering::SeqCst); + fast_stats.notify_one(); + } + http_ok(r#"{"tokens_per_second":42.0}"#) + } else { + unexpected.fetch_add(1, Ordering::Relaxed); + HTTP_503.to_string() + }; + let _ = socket.write_all(response.as_bytes()).await; + }); + } + })); + } + let (tx, mut rx) = broadcast::channel(64); + let clock = dir.path().join("clock"); + std::fs::write(&clock, "hold").unwrap(); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + discovery_tick: TICK, + instance_tick: TICK, + test_clock_offset_path: Some(clock), + ..RunnerOptions::default() + }, + ) + .await; + let overlap = wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 2).then(|| snap.instances.clone()) + }) + .await; + if overlap.is_err() { + task.abort(); + for server in &servers { + server.abort(); + } + } + let rows = overlap.expect("both registry rows appear in the first scrape snapshot"); + assert!( + rows.iter().all(|row| row.gen_tps == Some(42.0)), + "independent endpoint health and stats must overlap in the same cycle, not run serially: {rows:?}" + ); + drain_snapshots(&mut rx); + let previous_fast_stats = fast_stats_count.load(Ordering::SeqCst); + failed.store(true, Ordering::SeqCst); + // Allow fast health through while slow health is now failing immediately. + slow_started.notify_one(); + let failure = wait_for_snapshot(&mut rx, |snap| { + snap.warnings + .iter() + .any(|warning| { + warning.contains("lemonade health") + && warning.contains("slow") + && warning.contains("503") + }) + .then(|| snap.instances.clone()) + }) + .await; + task.abort(); + for server in servers { + server.abort(); + } + let rows = + failure.expect("health failures must produce endpoint/row-context snapshot warnings"); + assert!( + fast_stats_count.load(Ordering::SeqCst) > previous_fast_stats, + "independent fast endpoint completes another stats scrape during slow health failure" + ); + assert!( + rows.iter().all(|row| row.gen_tps == Some(42.0)), + "failed health holds existing rates; independent fast endpoint still scrapes" + ); + assert_eq!( + unexpected.load(Ordering::Relaxed), + 0, + "only health-before-stats reads are permitted" + ); +} + +/// A managed record whose identity (model or host) is rewritten while a rate is +/// attributed must drop that rate and its observation. The test clock is held so +/// expiry cannot clear it, and the stats route stops reporting a rate after the +/// rewrite so only the identity reset can remove the held value. +async fn assert_record_rewrite_resets_attributed_rate( + write_initial: fn(&std::path::Path, u16), + rewrite: fn(&std::path::Path, u16), +) { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let stats = Arc::new(Mutex::new(r#"{"tokens_per_second":42.0}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let listener = TcpListener::bind("0.0.0.0:0").await.unwrap(); + let (port, server) = + lemonade_fixture_on(listener, health, stats.clone(), unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + let record = dir.path().join("managed.json"); + write_initial(&record, port); + let clock_dir = tempfile::tempdir().unwrap(); + let clock = clock_dir.path().join("clock"); + std::fs::write(&clock, "hold").unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + test_clock_offset_path: Some(clock), + ..RunnerOptions::default() + }, + ) + .await; + let outcome = async { + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].gen_tps == Some(42.0)).then_some(()) + }) + .await + .map_err(|e| format!("rate attributed: {e}"))?; + // Control: with the stats rate gone and identity unchanged, the held + // value survives, so a later None can only come from the reset. + *stats.lock().unwrap() = "{}".to_string(); + for _ in 0..4 { + let held = wait_for_snapshot(&mut rx, |snap| Some(snap.instances.clone())).await?; + if held.len() != 1 || held[0].gen_tps != Some(42.0) { + return Err(format!( + "held rate lost without an identity change: {held:?}" + )); + } + } + rewrite(&record, port); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 + && snap.instances[0].gen_tps.is_none() + && snap.instances[0].gen_tps_observation.is_none()) + .then_some(()) + }) + .await + .map_err(|e| format!("identity rewrite must reset the attributed rate: {e}")) + } + .await; + task.abort(); + server.abort(); + outcome.unwrap(); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +fn write_managed_record(path: &std::path::Path, model_ref: &str, host: &str, port: u16) { + std::fs::write(path, format!(r#"{{"service_id":"managed-lemonade","engine":"lemonade","model_ref":"{model_ref}","canonical_model_id":"Model-A","host":"{host}","port":{port},"status":"ready"}}"#)).unwrap(); +} + +#[tokio::test] +async fn lemonade_managed_model_rewrite_resets_attributed_rate() { + assert_record_rewrite_resets_attributed_rate( + |path, port| write_managed_record(path, "alias-one", "127.0.0.1", port), + |path, port| write_managed_record(path, "alias-two", "127.0.0.1", port), + ) + .await; +} + +/// Rewriting `canonical_model_id` to a name health does not report leaves the row +/// matching nothing, so the attributed rate is cleared because attribution is +/// lost. The same-endpoint identity clause is isolated by +/// `lemonade_managed_canonical_rewrite_that_still_matches_health_resets_rate`. +#[tokio::test] +async fn lemonade_managed_canonical_rewrite_that_unmatches_health_clears_rate() { + assert_record_rewrite_resets_attributed_rate( + |path, port| write_managed_record(path, "alias-one", "127.0.0.1", port), + |path, port| { + write_managed_record(path, "alias-one", "127.0.0.1", port); + let mut record: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + record["canonical_model_id"] = serde_json::json!("Model-B"); + std::fs::write(path, serde_json::to_vec(&record).unwrap()).unwrap(); + }, + ) + .await; +} + +/// Only `canonical_model_id` changes, and the row still matches health through +/// its `model_ref`. Nothing else then clears the rate (the row is not +/// unmatched), so removing the canonical-id clause from the same-endpoint check +/// leaves the old rate attributed to the rewritten service. +#[tokio::test] +async fn lemonade_managed_canonical_rewrite_that_still_matches_health_resets_rate() { + fn write(path: &std::path::Path, canonical: &str, port: u16) { + std::fs::write(path, format!(r#"{{"service_id":"managed-lemonade","engine":"lemonade","model_ref":"Model-A","canonical_model_id":"{canonical}","host":"127.0.0.1","port":{port},"status":"ready"}}"#)).unwrap(); + } + assert_record_rewrite_resets_attributed_rate( + |path, port| write(path, "Model-A", port), + |path, port| write(path, "Model-B", port), + ) + .await; +} + +#[tokio::test] +async fn lemonade_managed_host_rewrite_resets_attributed_rate() { + assert_record_rewrite_resets_attributed_rate( + |path, port| write_managed_record(path, "alias-one", "127.0.0.1", port), + |path, port| write_managed_record(path, "alias-one", "127.0.0.2", port), + ) + .await; +} + +/// With discovery three times slower than scraping, a health failure on a +/// scrape-only tick holds the external rows and warns; the next discovery tick +/// with the same failure removes them. The fixture counts health requests (one +/// per tick) and fails from the fourth, so the first failing ticks are +/// deterministically scrape-only (discovery runs on ticks 1, 3, 6, ...). +/// +/// `failure` is the status the health route starts answering. A 401/403 must +/// behave like the 503: the external endpoint is checked as strictly on a +/// scrape-only tick as on a discovery tick, so the warning appears on the first +/// failing tick rather than waiting for the next discovery tick. +async fn scrape_only_health_failure_holds_external_rows(failure: &'static str) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let hits = Arc::new(AtomicU64::new(0)); + let server_hits = hits.clone(); + let server = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let hits = server_hits.clone(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + let response = if request.starts_with("GET /api/v1/health ") { + if hits.fetch_add(1, Ordering::SeqCst) + 1 >= 4 { + format!( + "HTTP/1.1 {failure}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ) + } else { + http_ok(r#"{"status":"ok","model_loaded":"Model-A"}"#) + } + } else if request.starts_with("GET /api/v1/stats ") { + http_ok(r#"{"tokens_per_second":42.0}"#) + } else { + HTTP_503.to_string() + }; + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + let (tx, mut rx) = broadcast::channel(256); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: 3 * TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let outcome = async { + let mut saw_held = false; + loop { + let (rows, warned) = wait_for_snapshot(&mut rx, |snap| { + Some(( + snap.instances.len(), + snap.warnings.iter().any(|w| w.contains("lemonade health")), + )) + }) + .await?; + match (rows, warned) { + (1, true) => saw_held = true, + (0, true) => { + return if saw_held { + Ok(()) + } else { + Err("rows removed by a scrape-only tick".to_string()) + }; + } + (0, false) if hits.load(Ordering::SeqCst) >= 4 => { + return Err("rows removed without a warning".to_string()); + } + _ => {} + } + } + } + .await; + task.abort(); + server.abort(); + outcome.unwrap(); +} + +#[tokio::test] +async fn lemonade_scrape_only_tick_holds_external_rows_until_next_discovery() { + scrape_only_health_failure_holds_external_rows("503 Service Unavailable").await; +} + +/// An already-known external endpoint that starts requiring credentials is +/// reported on the scrape-only tick that finds out, not one discovery interval +/// later (the managed-target tolerance for 401/403 does not apply to it). +#[tokio::test] +async fn lemonade_scrape_only_tick_warns_when_a_known_external_endpoint_requires_credentials() { + for status in ["401 Unauthorized", "403 Forbidden"] { + scrape_only_health_failure_holds_external_rows(status).await; + } +} + +/// A known external endpoint whose stats route fails while health is fine and one +/// model is loaded (so the rate is attributable): the failure is reported, the row +/// stays, and nothing but reads reaches the server. +#[tokio::test] +async fn lemonade_stats_failure_on_an_attributed_endpoint_warns_and_keeps_the_row() { + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let stats = Arc::new(Mutex::new("HTTP_503".to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (port, server) = lemonade_fixture_on(listener, health, stats, unexpected.clone()).await; + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let seen = wait_for_snapshot(&mut rx, |snap| { + snap.warnings + .iter() + .any(|w| w.contains("lemonade stats")) + .then(|| (snap.instances.clone(), snap.warnings.clone())) + }) + .await; + task.abort(); + server.abort(); + let (rows, warnings) = seen.expect("a stats failure is reported"); + assert_eq!(rows.len(), 1, "the row stays listed: {rows:?}"); + assert!(rows[0].gen_tps.is_none(), "{rows:?}"); + assert!( + warnings.iter().any(|w| w.contains("lemonade stats") + && w.contains(&port.to_string()) + && w.contains("503")), + "the warning names the endpoint and the status: {warnings:?}" + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +/// A managed target whose health route answers with a genuine 5xx (not the +/// 404/401/403 that mean "no router" or "credentials needed") is a failure to +/// report: the registry row stays with its registry status, and the warning +/// names the endpoint and the managed service. +#[tokio::test] +async fn lemonade_managed_endpoint_health_5xx_warns_and_keeps_the_registry_row() { + let health = Arc::new(Mutex::new("HTTP_503".to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture(health, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("managed.json"), format!(r#"{{"service_id":"managed-5xx","engine":"lemonade","model_ref":"Model-A","host":"127.0.0.1","port":{port},"status":"ready"}}"#)).unwrap(); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let seen = wait_for_snapshot(&mut rx, |snap| { + snap.warnings + .iter() + .any(|w| w.contains("lemonade health")) + .then(|| (snap.instances.clone(), snap.warnings.clone())) + }) + .await; + task.abort(); + server.abort(); + let (rows, warnings) = seen.expect("a managed health 5xx is reported"); + assert_eq!(rows.len(), 1, "{rows:?}"); + assert!( + rows[0].managed && rows[0].container_id == "managed-5xx", + "{rows:?}" + ); + assert!(rows[0].gen_tps.is_none(), "{rows:?}"); + assert!( + warnings.iter().any(|w| w.contains("lemonade health") + && w.contains(&port.to_string()) + && w.contains("managed-5xx") + && w.contains("503")), + "{warnings:?}" + ); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +/// Bind an IPv6 listener for a test that exercises IPv6 handling. +/// +/// A failed bind fails the test: a bare `return` reads as a pass, and these two +/// tests are the only cover for the `::` to `::1` mapping and for the collector +/// cache's reuse across address families. A machine with no IPv6 loopback opts +/// out with `ROCM_TEST_NO_IPV6=1`, which prints that the test did not run. +async fn bind_ipv6(addr: &str) -> Option { + match TcpListener::bind(addr).await { + Ok(listener) => Some(listener), + Err(error) if std::env::var_os("ROCM_TEST_NO_IPV6").is_some() => { + eprintln!("SKIPPED (ROCM_TEST_NO_IPV6 set): {addr} bind failed: {error}"); + None + } + Err(error) => panic!( + "IPv6 bind {addr} failed: {error}; this test needs IPv6 loopback (set ROCM_TEST_NO_IPV6=1 on a machine that has none)" + ), + } +} + +/// A dual-stack `::` bind at the external endpoint's port is the same service as +/// the IPv4 external probe; it must not also appear as an external row. +#[tokio::test] +async fn lemonade_dual_stack_managed_bind_does_not_duplicate_external_row() { + let Some(listener) = bind_ipv6("[::]:0").await else { + return; + }; + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let stats = Arc::new(Mutex::new(r#"{"tokens_per_second":42.0}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (port, server) = lemonade_fixture_on(listener, health, stats, unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + write_managed_record(&dir.path().join("managed.json"), "Model-A", "::", port); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: true, + lemonade_port: port, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let outcome = async { + wait_for_snapshot(&mut rx, |snap| { + snap.instances + .iter() + .any(|row| row.container_id == "managed-lemonade" && row.gen_tps == Some(42.0)) + .then_some(()) + }) + .await?; + for _ in 0..5 { + let rows = wait_for_snapshot(&mut rx, |snap| Some(snap.instances.clone())).await?; + if rows.len() != 1 || rows[0].container_id != "managed-lemonade" { + return Err(format!("duplicate external row: {rows:?}")); + } + } + Ok(()) + } + .await; + task.abort(); + server.abort(); + outcome.unwrap(); + assert_eq!(unexpected.load(Ordering::Relaxed), 0); +} + +/// A managed `::` record replaced by a `127.0.0.1` record on the same port +/// must not keep dialing the old IPv6 address: the IPv4-only listener has to +/// come up healthy with no standing warning. +#[tokio::test] +async fn lemonade_managed_ipv6_record_replaced_by_ipv4_record_on_same_port_recovers() { + // Take an IPv6 loopback port, then the same port on IPv4. Another process + // can hold the IPv4 side of a given port, so try a few ports before giving up. + let mut pair = None; + for _ in 0..20 { + let Some(v6) = bind_ipv6("[::1]:0").await else { + return; + }; + let port = v6.local_addr().unwrap().port(); + if let Ok(v4) = TcpListener::bind(("127.0.0.1", port)).await { + pair = Some((v6, v4, port)); + break; + } + } + let (v6, v4, port) = pair.expect("no port free on both IPv6 and IPv4 loopback after 20 tries"); + let health = Arc::new(Mutex::new( + r#"{"status":"ok","model_loaded":"Model-A"}"#.to_string(), + )); + let stats = Arc::new(Mutex::new(r#"{"tokens_per_second":42.0}"#.to_string())); + let unexpected = Arc::new(AtomicU64::new(0)); + let (_, v6_server) = + lemonade_fixture_on(v6, health.clone(), stats.clone(), unexpected.clone()).await; + let dir = tempfile::tempdir().unwrap(); + let record = dir.path().join("managed.json"); + write_managed_record(&record, "Model-A", "::", port); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let outcome = async { + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].gen_tps == Some(42.0)).then_some(()) + }) + .await + .map_err(|e| format!("ipv6 record healthy: {e}"))?; + // The old listener goes away and an IPv4-only service takes the port. + v6_server.abort(); + let _ = v6_server.await; + let (_, _v4_server) = + lemonade_fixture_on(v4, health.clone(), stats.clone(), unexpected.clone()).await; + // The replacement lands in one step, so no discovery tick sees an empty + // registry that would drop the cached collector on its own. + write_managed_record(&record, "Model-A", "127.0.0.1", port); + wait_for_snapshot(&mut rx, |snap| { + (snap.instances.len() == 1 && snap.instances[0].gen_tps == Some(42.0)).then_some(()) + }) + .await + .map_err(|e| format!("ipv4 record healthy: {e}"))?; + for _ in 0..4 { + let warnings = wait_for_snapshot(&mut rx, |snap| Some(snap.warnings.clone())).await?; + if warnings.iter().any(|w| w.contains("lemonade")) { + return Err(format!("standing warning after takeover: {warnings:?}")); + } + } + Ok::<(), String>(()) + } + .await; + task.abort(); + outcome.unwrap(); +} + +/// A managed direct llama-server row (the server answers only `/health` and +/// `/v1/models`, with no stats route) stays listed across repeated scrapes with +/// `gen_tps` left at `None` and no standing Lemonade warning. +#[tokio::test] +async fn lemonade_managed_direct_llama_server_row_has_no_standing_warning() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + tokio::spawn(async move { + let mut bytes = [0; 4096]; + let n = socket.read(&mut bytes).await.unwrap(); + let request = String::from_utf8_lossy(&bytes[..n]); + let response = if request.starts_with("GET /health ") { + http_ok(r#"{"status":"ok"}"#) + } else if request.starts_with("GET /v1/models ") { + http_ok(r#"{"data":[{"id":"Model-A"}]}"#) + } else { + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + .to_string() + }; + let _ = socket.write_all(response.as_bytes()).await; + }); + } + }); + let dir = tempfile::tempdir().unwrap(); + write_managed_record( + &dir.path().join("direct.json"), + "Model-A", + "127.0.0.1", + port, + ); + let (tx, mut rx) = broadcast::channel(64); + let task = spawn_runner( + tx, + RunnerOptions { + services_dir: Some(dir.path().to_path_buf()), + enable_lemonade: false, + discovery_tick: TICK, + instance_tick: TICK, + ..RunnerOptions::default() + }, + ) + .await; + let outcome = async { + wait_for_snapshot(&mut rx, |snap| (snap.instances.len() == 1).then_some(())).await?; + for _ in 0..6 { + let (rows, warnings) = wait_for_snapshot(&mut rx, |snap| { + Some((snap.instances.clone(), snap.warnings.clone())) + }) + .await?; + if rows.len() != 1 || rows[0].gen_tps.is_some() { + return Err(format!("row must stay visible without a rate: {rows:?}")); + } + if warnings.iter().any(|w| w.contains("lemonade")) { + return Err(format!("standing warning on a direct server: {warnings:?}")); + } + } + Ok(()) + } + .await; + task.abort(); + server.abort(); + outcome.unwrap(); +} + // ── Timing constants ──────────────────────────────────────────────────────── const TICK: Duration = Duration::from_millis(200); diff --git a/crates/rocm-dash-tui/src/agent/snapshot.rs b/crates/rocm-dash-tui/src/agent/snapshot.rs index 27d086774c..fd1e071bab 100644 --- a/crates/rocm-dash-tui/src/agent/snapshot.rs +++ b/crates/rocm-dash-tui/src/agent/snapshot.rs @@ -80,7 +80,11 @@ fn freshness_label(freshness: Option) -> Value { /// freshness metadata, and observed_at timestamp. /// /// Fields added by EAI-7960: -/// - `gen_tps`: live generation throughput from the Instance (same as daemon) +/// - `gen_tps`: generation throughput from the Instance (same as daemon) +/// - `gen_tps_last_request`: `true` when `gen_tps` is only the endpoint's most +/// recent completed request (Lemonade), which stays unchanged while the server +/// is idle, so it must not be read as live throughput; `tokens_per_watt` is +/// derived from it /// - `tokens_per_watt`: daemon-provided efficiency (never recomputed here) /// - `attributed_power_w`: this instance's share of its GPUs' board power /// (split among instances sharing a GPU). Not the same as `gpu_status`'s @@ -117,6 +121,7 @@ pub fn list_instances_json(snap: &StateSnapshot) -> Value { "running_reqs": i.running_reqs, "waiting_reqs": i.waiting_reqs, "gen_tps": i.gen_tps, + "gen_tps_last_request": i.gen_tps_last_request, "tokens_per_watt": i.tokens_per_watt, "attributed_power_w": i.power_w, "ttft_ms": i.ttft_ms, @@ -164,6 +169,7 @@ pub fn tokens_per_watt_json(snap: &StateSnapshot) -> Value { json!({ "name": i.container_name, "gen_tps": i.gen_tps, + "gen_tps_last_request": i.gen_tps_last_request, "tokens_per_watt": i.tokens_per_watt, "freshness": freshness, "observed_at": observed_at, @@ -349,6 +355,23 @@ mod tests { ); } + /// A Lemonade rate is the last completed request's, not live throughput; the + /// agent must be able to tell, in both tools that report `gen_tps`. + #[test] + fn instance_tools_carry_the_last_request_flag() { + for last_request in [true, false] { + let mut snap = + fixture_with_observation(rocm_dash_core::metrics::ObservationFreshness::Fresh); + snap.instances[0].gen_tps_last_request = last_request; + for v in [list_instances_json(&snap), tokens_per_watt_json(&snap)] { + assert_eq!( + v["instances"][0]["gen_tps_last_request"], last_request, + "{v}" + ); + } + } + } + #[test] fn list_instances_json_reports_latency_freshness_and_attributed_power() { use chrono::Utc; diff --git a/crates/rocm-dash-tui/src/app/event_loop.rs b/crates/rocm-dash-tui/src/app/event_loop.rs index 8709b4a53e..4f19f13f71 100644 --- a/crates/rocm-dash-tui/src/app/event_loop.rs +++ b/crates/rocm-dash-tui/src/app/event_loop.rs @@ -234,6 +234,8 @@ async fn event_loop(terminal: &mut Tui, args: &ResolvedArgs) -> color_eyre::Resu args.managed_registry_probe.clone(), args.launcher_hosted, ); + // Managed Lemonade install inventory, captured by the bin at launch. + state.lemonade_managed_installed = args.lemonade_managed_installed; // Focused host: open exactly the overlay for the requested flow (Examine // also auto-runs its read-only job). `Focus::Setup` opens the onboarding // overlay — the same wizard `rocm bootstrap setup` routes to. diff --git a/crates/rocm-dash-tui/src/app/mod.rs b/crates/rocm-dash-tui/src/app/mod.rs index 936fd56bb2..e6d577ed9b 100644 --- a/crates/rocm-dash-tui/src/app/mod.rs +++ b/crates/rocm-dash-tui/src/app/mod.rs @@ -249,6 +249,8 @@ pub struct AppState { /// that actually exits the process. See the field doc on /// `ResolvedArgs::launcher_hosted` and `actions::request_quit`. pub(crate) launcher_hosted: bool, + /// Managed installation inventory captured at launch; not endpoint liveness. + pub lemonade_managed_installed: Option, /// Last body area used by the most recent draw. Mouse hit-tests resolve /// pointer coordinates against this rect (filled by `ui::draw`). pub last_body_area: Option, @@ -416,6 +418,7 @@ impl AppState { services_past_attempts: 0, managed_registry_probe: None, launcher_hosted: false, + lemonade_managed_installed: None, last_body_area: None, last_tab_bar_area: None, last_header_badge: None, diff --git a/crates/rocm-dash-tui/src/app/quit_confirm.rs b/crates/rocm-dash-tui/src/app/quit_confirm.rs index c0c6f7096e..efbb2e340b 100644 --- a/crates/rocm-dash-tui/src/app/quit_confirm.rs +++ b/crates/rocm-dash-tui/src/app/quit_confirm.rs @@ -104,13 +104,25 @@ impl AppState { /// call sites also ask a sibling registry-backed question in the same /// decision and need the two answered from one read, not two independent /// ones a registry write in between could answer differently. + /// + /// Both halves of ownership must hold: the live registry read names the + /// instance, and the daemon's broadcast already marks it `managed`, because + /// the Services overlay lists only broadcast-managed rows + /// ([`is_listed_service`](crate::ui::services_manager::is_listed_service)). + /// An instance started a moment ago can be on disk before the next broadcast + /// says so; it is not offered here, because `s` could not focus a row for it. + /// The quit prompt still prints `rocm services stop --yes` for it (that + /// line keys off the registry alone), and the offer appears on the broadcast + /// that catches up. pub(crate) fn has_stoppable_managed_instance_with( &self, snapshot: &crate::registry_probe::ManagedRegistrySnapshot, ) -> bool { - self.instances - .values() - .any(|i| i.status.is_serving() && snapshot.managed_ids.contains(&i.container_id)) + self.instances.values().any(|i| { + crate::ui::services_manager::is_listed_service(i) + && i.status.is_serving() + && snapshot.managed_ids.contains(&i.container_id) + }) } /// Whether the confirm-before-quit prompt is open. Mirrors @@ -194,13 +206,15 @@ impl AppState { /// that normal resolution instead. /// /// The short-circuit above calls - /// [`has_stoppable_managed_instance_with`](Self::has_stoppable_managed_instance_with) - /// on exactly the same predicate the row-finding `position` closure below - /// re-checks per row (`is_serving() && managed_ids.contains(..)`), so it - /// cannot change which outcome this returns — only skip it sooner. Its - /// value is avoiding `service_rows`'s allocation and sort on every `s` - /// keypress when nothing is stoppable; it is not a correctness guard, and - /// there is nothing for the two checks to drift on. + /// [`has_stoppable_managed_instance_with`](Self::has_stoppable_managed_instance_with), + /// which applies the same ownership test `service_rows` lists by + /// (`is_listed_service`) plus the per-row check the `position` closure below + /// repeats (`is_serving() && managed_ids.contains(..)`). Because `service_rows` + /// lists only broadcast-managed instances, a predicate without that test would + /// say "stoppable" for an instance the rows do not contain and the key would + /// be dropped silently, so the two must keep agreeing; the test + /// `a_registry_managed_instance_the_broadcast_does_not_yet_mark_managed_is_not_offered` + /// pins that. pub(crate) fn quit_confirm_open_services_key( &mut self, code: crossterm::event::KeyCode, @@ -369,6 +383,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status, ..Default::default() }, @@ -400,6 +415,7 @@ mod tests { "svc-a".to_string(), Instance { container_id: "svc-a".to_string(), + managed: true, status: InstanceStatus::Stopped, ..Default::default() }, @@ -408,6 +424,7 @@ mod tests { "svc-b".to_string(), Instance { container_id: "svc-b".to_string(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, @@ -440,6 +457,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status, ..Default::default() }, @@ -473,6 +491,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, @@ -519,6 +538,42 @@ mod tests { assert!(s.services.is_none()); } + /// A managed instance started a moment ago is on disk (`managed_ids`) before the + /// daemon's next broadcast marks it `managed`. The Services overlay lists only + /// broadcast-managed rows, so `s` cannot focus one for it: the stop offer must + /// not be made, and `s` must say it did not handle the key (so the prompt + /// resolves it normally) instead of claiming a stoppable instance and dropping it. + #[test] + fn a_registry_managed_instance_the_broadcast_does_not_yet_mark_managed_is_not_offered() { + use crossterm::event::KeyCode; + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + for (managed, offered) in [(false, false), (true, true)] { + let mut s = st(); + s.instances.insert( + "svc-new".to_string(), + Instance { + container_id: "svc-new".to_string(), + managed, + status: InstanceStatus::Ready, + ..Default::default() + }, + ); + with_registry_probe(&mut s, true, &["svc-new"]); + assert_eq!( + s.has_stoppable_managed_instance_with(&s.live_registry_snapshot()), + offered, + "managed={managed}" + ); + s.open_quit_confirm(); + assert_eq!( + s.quit_confirm_open_services_key(KeyCode::Char('s')), + offered, + "managed={managed}" + ); + assert_eq!(s.services.is_some(), offered, "managed={managed}"); + } + } + #[test] fn quit_confirm_open_services_key_ignores_other_keys_and_a_closed_prompt() { use crossterm::event::KeyCode; diff --git a/crates/rocm-dash-tui/src/app/test_support.rs b/crates/rocm-dash-tui/src/app/test_support.rs index 0920e4a796..26e2df4aac 100644 --- a/crates/rocm-dash-tui/src/app/test_support.rs +++ b/crates/rocm-dash-tui/src/app/test_support.rs @@ -71,5 +71,6 @@ pub(super) fn args_with_anthropic_key(key: Option<&str>) -> ResolvedArgs { services_past_attempts: 0, managed_registry_probe: None, launcher_hosted: false, + lemonade_managed_installed: None, } } diff --git a/crates/rocm-dash-tui/src/app/types.rs b/crates/rocm-dash-tui/src/app/types.rs index 27fb3962ae..1476b25d24 100644 --- a/crates/rocm-dash-tui/src/app/types.rs +++ b/crates/rocm-dash-tui/src/app/types.rs @@ -143,6 +143,10 @@ pub struct ResolvedArgs { /// serves still warns (see dash-29). See /// `registry_probe::ManagedRegistryProbe`. pub managed_registry_probe: Option, + /// Launch-time managed Lemonade installation verdict from the bin's built-in + /// Detect method. None means unavailable/unknown, never not installed. + /// Separate from live external-serving telemetry and GPU system metadata. + pub lemonade_managed_installed: Option, } impl ResolvedArgs { diff --git a/crates/rocm-dash-tui/src/ui/format.rs b/crates/rocm-dash-tui/src/ui/format.rs index 0f8fe72c2b..01eb031621 100644 --- a/crates/rocm-dash-tui/src/ui/format.rs +++ b/crates/rocm-dash-tui/src/ui/format.rs @@ -230,6 +230,30 @@ pub const HELD_MARKER: &str = "*"; /// panels rather than inventing view-specific descriptions. pub const HELD_LEGEND: &str = "* = held (prior scrape window)"; +/// Explains the tok/s of a Lemonade row: the endpoint reports the rate of its most +/// recent completed request, which stays unchanged while the server is idle. +/// +/// Short enough to join the held legend inside an 80-column table. +pub const LAST_REQ_LEGEND: &str = "Lemonade: last req tok/s, not live"; + +/// The label for a throughput figure (compact rows and the instance detail). +/// +/// `last req` for the rate of a Lemonade endpoint's most recent request +/// ([`rocm_dash_core::metrics::Instance::gen_tps_last_request`]), `gen` for a +/// live rate. +#[must_use] +pub const fn gen_rate_label(last_request: bool) -> &'static str { + if last_request { "last req" } else { "gen" } +} + +/// [`gen_rate_label`] for a column with a fixed budget: as wide as `gen`, so a +/// last-request rate is never clipped to a different number. A legend +/// ([`LAST_REQ_LEGEND`]) explains it. +#[must_use] +pub const fn gen_rate_label_short(last_request: bool) -> &'static str { + if last_request { "req" } else { "gen" } +} + /// Compact gen_tps for narrow table cells (matches the existing bare-number format). /// /// - `None` → `"—"` (unavailable; em-dash matches the rest of the table) diff --git a/crates/rocm-dash-tui/src/ui/instance_detail.rs b/crates/rocm-dash-tui/src/ui/instance_detail.rs index 41b24be0e2..ab3871d117 100644 --- a/crates/rocm-dash-tui/src/ui/instance_detail.rs +++ b/crates/rocm-dash-tui/src/ui/instance_detail.rs @@ -63,25 +63,35 @@ pub(super) fn draw_detail(f: &mut Frame, area: Rect, state: &AppState, theme: &T return 0; } - // Vertical: summary (4 lines: status/id/port/tp · model/gpus/tpw/gen · partition/quant/vram · freshness) + // Vertical: summary (4 lines: status/id/port/tp · model/gpus/tpw/gen · partition/quant/vram · freshness, + // taller when a line wraps at this width: the last line is the one that + // qualifies the rate, so it must never be what falls off) // | recent throughput (recent non-idle ticks, ROCMAI-196) | body (min) | footer (1) let snap_ts = state.latest.as_ref().map(|s| s.timestamp); let activity = recent_activity_rows(&state.history, &inst.container_id, RECENT_ACTIVITY_CAP); let has_observed_throughput = has_observed_throughput(&state.history, &inst.container_id); - let (activity_h, visible_rows) = activity_layout(activity.len(), inner.height); + let summary = summary_lines(inner.width, inst, snap_ts, theme); + let summary_h = u16::try_from( + Paragraph::new(summary.clone()) + .wrap(Wrap { trim: false }) + .line_count(inner.width), + ) + .unwrap_or(u16::MAX) + .clamp(SUMMARY_ROWS, (inner.height / 2).max(SUMMARY_ROWS)); + let (activity_h, visible_rows) = activity_layout(activity.len(), inner.height, summary_h); let total_rows = activity.len(); let activity = &activity[..visible_rows]; let chunks = Layout::default() .direction(Direction::Vertical) .constraints([ - Constraint::Length(4), + Constraint::Length(summary_h), Constraint::Length(activity_h), Constraint::Min(0), Constraint::Length(1), ]) .split(inner); - render_summary(f, chunks[0], inst, snap_ts, theme); + render_summary(f, chunks[0], summary); render_recent_activity( f, chunks[1], @@ -96,6 +106,9 @@ pub(super) fn draw_detail(f: &mut Frame, area: Rect, state: &AppState, theme: &T max_scroll } +/// Rows of the summary block when none of its lines wraps. +const SUMMARY_ROWS: u16 = 4; + /// Max rows shown in the detail modal's "recent throughput" summary. const RECENT_ACTIVITY_CAP: usize = 5; @@ -120,8 +133,8 @@ fn body_floor(inner_h: u16) -> u16 { /// budget, so a capped panel shows fewer (newest) rows instead of being /// handed rows it can't draw. A panel too short for its header plus one row /// (or the placeholder line) collapses to `(0, 0)` so the body gets the rows. -fn activity_layout(n_rows: usize, inner_h: u16) -> (u16, usize) { - let avail = inner_h.saturating_sub(4 + 1 + body_floor(inner_h)); +fn activity_layout(n_rows: usize, inner_h: u16, summary_h: u16) -> (u16, usize) { + let avail = inner_h.saturating_sub(summary_h + 1 + body_floor(inner_h)); // 2 border rows + (1 header row + 1 per entry), or 1 placeholder row when // there's none yet; `panel::bento` adds a top-padding row once the box is // tall enough (`padding_for`), which is reused here rather than re-derived. @@ -153,7 +166,10 @@ pub(crate) struct ActivityRow { /// `container_id` had non-zero generation throughput — i.e. was actually /// serving, not idle. These are per-tick samples, not discrete serving runs. /// The live "gen:" gauge in `render_summary` already covers the idle case, so -/// idle ticks are skipped. Repeated Held ticks of one scrape collapse to one row. Returns up to `cap` rows, newest first. +/// idle ticks are skipped. A Lemonade last-request rate +/// ([`rocm_dash_core::metrics::Instance::gen_tps_last_request`]) is not serving +/// either: it is one finished request's rate re-reported on every tick, so it is +/// skipped like an idle tick. Repeated Held ticks of one scrape collapse to one row. Returns up to `cap` rows, newest first. /// Pure — exposed for tests. pub(crate) fn recent_activity_rows( history: &std::collections::VecDeque, @@ -169,7 +185,7 @@ pub(crate) fn recent_activity_rows( else { continue; }; - if !inst.gen_tps.is_some_and(|v| v.is_finite() && v > 0.0) { + if !inst.live_gen_tps().is_some_and(|v| v > 0.0) { continue; } // A Held tick repeats an earlier scrape; age it from when the @@ -297,18 +313,17 @@ fn render_recent_activity( f.render_widget(table, inner); } -fn render_summary( - f: &mut Frame, - area: Rect, +fn summary_lines( + width: u16, inst: &Instance, snap_ts: Option>, theme: &Theme, -) { +) -> Vec> { let (status_color, status_text) = status_meta(inst.status, theme); let muted = Style::default().fg(theme.muted); let fg = Style::default().fg(theme.fg); - let id_w = (area.width as usize).saturating_sub(16).max(8); + let id_w = (width as usize).saturating_sub(16).max(8); let container_id = trunc(&inst.container_id, id_w); let partition = inst.partition_info.as_deref().unwrap_or("-"); let quant = inst.quantization.as_deref().unwrap_or("-"); @@ -356,13 +371,16 @@ fn render_summary( Span::styled("tok/W: ", muted), Span::styled( format::tokens_per_watt_cell( - inst.tokens_per_watt, + inst.live_tokens_per_watt(), inst.gen_tps_observation.as_ref(), ), Style::default().fg(theme.accent), ), Span::raw(" "), - Span::styled("gen: ", muted), + Span::styled( + format!("{}: ", format::gen_rate_label(inst.gen_tps_last_request)), + muted, + ), Span::styled( format::gen_tps_compact(inst.gen_tps, inst.gen_tps_observation.as_ref()), fg, @@ -387,7 +405,10 @@ fn render_summary( Span::styled(observed_str, Style::default().fg(theme.muted)), ]), ]; + lines +} +fn render_summary(f: &mut Frame, area: Rect, lines: Vec>) { let p = Paragraph::new(lines).wrap(Wrap { trim: false }); f.render_widget(p, area); } @@ -600,7 +621,8 @@ fn render_footer(f: &mut Frame, area: Rect, inst: &Instance, theme: &Theme, scro mod tests { use super::*; use crate::ui::tabs::instances::tests::{ - SNAP_SECS, buffer_text, mk_inst, mk_inst_obs, mk_snap, mk_state, obs, state_with_snap, + SNAP_SECS, buffer_text, last_req_inst, mk_inst, mk_inst_obs, mk_snap, mk_state, obs, + state_with_snap, }; use chrono::TimeZone; use rocm_dash_core::metrics::{ObservationFreshness, ObservationMetadata}; @@ -1018,6 +1040,56 @@ mod tests { ); } + #[test] + fn detail_names_a_last_request_rate() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + for (last_request, label) in [(true, "last req: 37.5 tok/s"), (false, "gen: 37.5 tok/s")] { + let state = state_with_snap(last_req_inst("rate-detail", last_request)); + let mut term = Terminal::new(TestBackend::new(160, 48)).unwrap(); + term.draw(|f| { + draw_detail(f, f.area(), &state, &state.theme); + }) + .unwrap(); + let out = buffer_text(&term); + assert!(out.contains(label), "{label:?} missing:\n{out}"); + assert_eq!(out.contains("last req:"), last_request, "{out}"); + // The value already carries its unit, so the label must not repeat it. + assert!(!out.contains("tok/s:"), "unit doubled:\n{out}"); + } + } + + /// The summary block grows with a wrapped line: at no width may the line that + /// qualifies the rate (freshness and age) fall off the bottom, whatever the + /// rate's label length (`last req:` once pushed it out at 97-102 columns). + #[test] + fn detail_keeps_the_freshness_line_at_every_width() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + for last_request in [true, false] { + let mut inst = last_req_inst("a-realistic-container-id-0123456789", last_request); + inst.model_name = "gpt-oss-20b-mxfp4-instruct".into(); + inst.gpu_ids = vec!["0".into(), "1".into()]; + inst.partition_info = Some("SPX/NPS1".into()); + let state = state_with_snap(inst); + for width in 80..=200u16 { + let mut term = Terminal::new(TestBackend::new(width, 48)).unwrap(); + term.draw(|f| { + draw_detail(f, f.area(), &state, &state.theme); + }) + .unwrap(); + let out = buffer_text(&term); + assert!( + out.contains("gen freshness:") && out.contains("observed:"), + "last_request={last_request} width={width}: +{out}" + ); + } + } + } + #[test] fn detail_held_shows_held_freshness_label() { use ratatui::Terminal; @@ -1151,6 +1223,35 @@ mod tests { assert_eq!(rows[1].gen_tps, Some(10.0)); } + /// An idle Lemonade server keeps reporting its last request's rate. That is + /// data (so the panel says "idle", not "no throughput data"), but it is not + /// activity, and must not be listed as an unlabelled tok/s sample. + #[test] + fn a_last_request_rate_is_not_listed_as_recent_activity() { + let mut idle_lemonade = mk_inst("x"); + idle_lemonade.gen_tps = Some(45.0); + idle_lemonade.gen_tps_last_request = true; + let mut history: std::collections::VecDeque<_> = std::collections::VecDeque::new(); + for tick in 0..3 { + let mut snap = mk_snap(vec![idle_lemonade.clone()]); + snap.timestamp = Utc.timestamp_opt(1_700_000_000 + 30 * tick, 0).unwrap(); + history.push_back(snap); + } + assert!(recent_activity_rows(&history, "id-x", RECENT_ACTIVITY_CAP).is_empty()); + assert!( + has_observed_throughput(&history, "id-x"), + "the rate was reported, so the panel reads idle, not no data" + ); + // The same readings as a live rate are activity. + idle_lemonade.gen_tps_last_request = false; + let mut live = std::collections::VecDeque::new(); + live.push_back(mk_snap(vec![idle_lemonade])); + assert_eq!( + recent_activity_rows(&live, "id-x", RECENT_ACTIVITY_CAP).len(), + 1 + ); + } + #[test] fn recent_activity_rows_caps_to_requested_len() { let mut history: std::collections::VecDeque<_> = std::collections::VecDeque::new(); @@ -1529,8 +1630,8 @@ mod tests { fn activity_layout_never_exceeds_budget_and_keeps_newest_rows() { for inner_h in 0..=60u16 { for n in 0..=RECENT_ACTIVITY_CAP { - let (h, k) = activity_layout(n, inner_h); - let avail = inner_h.saturating_sub(4 + 1 + body_floor(inner_h)); + let (h, k) = activity_layout(n, inner_h, SUMMARY_ROWS); + let avail = inner_h.saturating_sub(SUMMARY_ROWS + 1 + body_floor(inner_h)); assert!(h <= avail, "n={n} inner_h={inner_h}: h={h} > {avail}"); assert!(k <= n); if n > 0 && h > 0 { @@ -1543,16 +1644,16 @@ mod tests { } } // Spacious: all five rows. Mid band: partial, but sized to match. - assert_eq!(activity_layout(5, 48).1, 5); - let (h, k) = activity_layout(5, 24); + assert_eq!(activity_layout(5, 48, SUMMARY_ROWS).1, 5); + let (h, k) = activity_layout(5, 24, SUMMARY_ROWS); assert!( (1..5).contains(&k), "mid-height band shows a partial panel: {k}" ); assert!(h > 0); // Placeholder: shown when it fits, collapsed otherwise. - assert_eq!(activity_layout(0, 48), (3, 0)); - assert_eq!(activity_layout(0, 8), (0, 0)); + assert_eq!(activity_layout(0, 48, SUMMARY_ROWS), (3, 0)); + assert_eq!(activity_layout(0, 8, SUMMARY_ROWS), (0, 0)); } #[test] diff --git a/crates/rocm-dash-tui/src/ui/launcher.rs b/crates/rocm-dash-tui/src/ui/launcher.rs index 346c4c746b..c1c8c57d43 100644 --- a/crates/rocm-dash-tui/src/ui/launcher.rs +++ b/crates/rocm-dash-tui/src/ui/launcher.rs @@ -879,6 +879,7 @@ mod tests { "vllm-1".into(), Instance { container_id: "vllm-1".into(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, diff --git a/crates/rocm-dash-tui/src/ui/mod.rs b/crates/rocm-dash-tui/src/ui/mod.rs index d3fd972891..367de2f173 100644 --- a/crates/rocm-dash-tui/src/ui/mod.rs +++ b/crates/rocm-dash-tui/src/ui/mod.rs @@ -464,6 +464,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, diff --git a/crates/rocm-dash-tui/src/ui/quit_confirm.rs b/crates/rocm-dash-tui/src/ui/quit_confirm.rs index 3e55a03361..f9ebaa4aa9 100644 --- a/crates/rocm-dash-tui/src/ui/quit_confirm.rs +++ b/crates/rocm-dash-tui/src/ui/quit_confirm.rs @@ -88,11 +88,11 @@ fn quit_confirm_body(state: &AppState) -> Vec { quit_confirm_body_with(state, &snapshot) } -/// Body text for the quit-confirm prompt: name the models still serving -/// (mirroring the serving rows of the Serving tab's "Running now" listing in -/// `tabs::pane::live_lines`'s `OpenServices` arm, as plain lines instead of -/// styled spans; that listing also shows starting services, which this gate -/// deliberately does not count), and how to stop each one that this host's own registry +/// Body text for the quit-confirm prompt: name every model still serving, +/// managed or not (the Serving tab's "Managed running" listing in +/// `tabs::pane::live_lines`'s `OpenServices` arm is managed-only, and also shows +/// starting services, which this gate deliberately does not count, so this +/// listing can be longer or shorter than it), and how to stop each one that this host's own registry /// manages (a live read, passed in as `snapshot` so the caller controls how /// many times the registry is actually read — see `quit_confirm_request`) — /// not a blanket command, since a Docker- or Lemonade-discovered instance @@ -190,10 +190,10 @@ pub(super) fn quit_confirm_body_with( // `values()` happened to yield, so the listing is at least stable // across renders. `s` does not focus "the Nth instance named here": // `quit_confirm_open_services_key` always focuses the FIRST managed, - // still-serving row in `service_rows`' own order, which can include - // an unmanaged (Docker/Lemonade) entry this listing also shows ahead - // of it — so the row `s` focuses need not be the Nth line printed - // below. + // still-serving row in `service_rows`' own order, and `service_rows` + // lists managed services only — so the row `s` focuses need not be the + // Nth line printed below, because this listing can also show an + // unmanaged (Docker/Lemonade) entry that the overlay does not. let mut sorted: Vec<_> = running().collect(); sorted.sort_by(|a, b| a.container_id.cmp(&b.container_id)); for i in sorted.into_iter().take(5) { @@ -379,6 +379,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, @@ -411,6 +412,41 @@ mod tests { assert_eq!(launcher.title, dashboard.title); } + /// Just started: the registry names the instance but the broadcast has not + /// marked it managed yet. The prompt must not offer `s` (the Services overlay + /// has no row for it); it still prints the command that stops it. + #[test] + fn quit_confirm_request_does_not_offer_s_for_an_instance_the_broadcast_has_not_marked_managed() + { + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + let mut state = AppState::new("t".into(), "default-dark".into()); + state.instances.insert( + "svc-new".to_string(), + Instance { + container_id: "svc-new".to_string(), + model_name: "m".to_string(), + managed: false, + status: InstanceStatus::Running, + ..Default::default() + }, + ); + with_registry_probe(&mut state, true, &["svc-new"]); + let request = quit_confirm_request(&state, true); + assert!( + !request.body.iter().any(|l| l.contains("Press s")), + "{:?}", + request.body + ); + assert!( + request + .body + .iter() + .any(|l| l.contains("rocm services stop svc-new --yes")), + "{:?}", + request.body + ); + } + #[test] fn quit_confirm_request_offers_the_stop_hint_only_for_the_dashboard_with_something_stoppable() { use rocm_dash_core::metrics::{Instance, InstanceStatus}; @@ -419,6 +455,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, status: InstanceStatus::Running, ..Default::default() }, @@ -481,6 +518,7 @@ mod tests { "svc-managed".to_string(), Instance { container_id: "svc-managed".to_string(), + managed: true, model_name: "Qwen2.5-7B".to_string(), status: InstanceStatus::Running, ..Default::default() @@ -563,8 +601,8 @@ mod tests { // `values()` happens to yield. This does not mean `s` // (`AppState::quit_confirm_open_services_key`) focuses the Nth // bullet printed here: it always focuses the FIRST managed, - // still-serving row in `service_rows`' own order, which can include - // an unmanaged entry this listing also shows. + // still-serving row in `service_rows`' own order, and `service_rows` + // lists managed services only. // `quit_confirm_body_caps_the_listing_at_five_with_an_overflow_line` // inserts ids (`svc-0`..`svc-5`) that already sort the same way // whether by id or insertion order, so it can't catch a regression diff --git a/crates/rocm-dash-tui/src/ui/services_manager.rs b/crates/rocm-dash-tui/src/ui/services_manager.rs index 1fe23796fe..4c6f429b91 100644 --- a/crates/rocm-dash-tui/src/ui/services_manager.rs +++ b/crates/rocm-dash-tui/src/ui/services_manager.rs @@ -5,8 +5,8 @@ //! Services manager overlay. //! //! It lists the managed inference services the daemon surfaces (model · port · -//! status · live `gen_tps`) and runs stop/restart **through the approval gate -//! and the job-bridge** — never inline. The serve wizard and the engine manager +//! status · live `gen_tps`, or the last request's rate) and runs stop/restart +//! **through the approval gate and the job-bridge** — never inline. The serve wizard and the engine manager //! follow the same pattern. //! //! Mutating actions invoke the CLI (`rocm services stop|restart --yes`) as @@ -90,14 +90,24 @@ pub struct ServiceRow { pub gen_tps: Option, /// Freshness metadata for `gen_tps`; `None` for legacy snapshots. pub gen_tps_observation: Option, + /// `gen_tps` is only the endpoint's most recent request, not a live rate. + pub gen_tps_last_request: bool, } -/// Build the sorted service list from the daemon-surfaced instances. +/// The one ownership predicate for lifecycle surfaces: only registry-owned +/// instances are listed and counted, because those are what the controls act on. +#[must_use] +pub const fn is_listed_service(i: &Instance) -> bool { + i.managed +} + +/// Build the sorted lifecycle list from registry-owned instances only. pub fn service_rows( instances: &HashMap, ) -> Vec { let mut rows: Vec = instances .values() + .filter(|i| is_listed_service(i)) .map(|i| ServiceRow { id: i.container_id.clone(), model: i.model_name.clone(), @@ -105,6 +115,7 @@ pub fn service_rows( status: i.status, gen_tps: i.gen_tps, gen_tps_observation: i.gen_tps_observation.clone(), + gen_tps_last_request: i.gen_tps_last_request, }) .collect(); rows.sort_by(|a, b| a.id.cmp(&b.id)); @@ -127,6 +138,20 @@ pub fn on_key( return Vec::new(); }; + // Discovery may remove the target or replace it with an external row while + // the modal is open. Revalidate ownership before handling any verdict. + // The key is not consumed: it falls through and is handled as a normal key. + if sm + .approval + .as_ref() + .is_some_and(|pending| !rows.iter().any(|row| row.id == pending.service_id)) + { + sm.approval = None; + } + + // Keep navigation aligned with the refreshed managed-only list. + sm.selected = sm.selected.min(rows.len().saturating_sub(1)); + // 1) Approval modal has focus. if let Some(pending) = sm.approval.as_mut() { let (choice, verdict) = approval_key(key.code, pending.choice); @@ -309,19 +334,24 @@ pub fn draw_services_manager( // it here introduces no circular dependency. let max_visible = inner.height.saturating_sub(1 + note_rows) as usize; let window = visible_list_window(selected, rows.len(), max_visible); - let any_held = rows.get(window).into_iter().flatten().any(|r| { + let any_held = rows.get(window.clone()).into_iter().flatten().any(|r| { r.gen_tps.is_some_and(f64::is_finite) && r.gen_tps_observation .as_ref() .is_some_and(|m| m.freshness == ObservationFreshness::Held) }); + let any_last_req = rows + .get(window) + .into_iter() + .flatten() + .any(|r| r.gen_tps_last_request && r.gen_tps.is_some_and(f64::is_finite)); let body = Layout::default() .direction(Direction::Vertical) .constraints([ Constraint::Min(1), Constraint::Length(note_rows), Constraint::Length(1), - Constraint::Length(u16::from(any_held)), + Constraint::Length(u16::from(any_held || any_last_req)), ]) .split(inner); @@ -359,7 +389,8 @@ pub fn draw_services_manager( ), Span::styled( format!( - "gen {}", + "{} {}", + format::gen_rate_label_short(r.gen_tps_last_request), format::gen_tps_compact(r.gen_tps, r.gen_tps_observation.as_ref()) ), Style::default().fg(theme.fg), @@ -403,10 +434,16 @@ pub fn draw_services_manager( body[2], ); - if any_held { + if any_held || any_last_req { f.render_widget( Paragraph::new(Line::from(Span::styled( - format::HELD_LEGEND, + match (any_held, any_last_req) { + (true, true) => { + format!("{} · {}", format::HELD_LEGEND, format::LAST_REQ_LEGEND) + } + (true, false) => format::HELD_LEGEND.to_owned(), + _ => format::LAST_REQ_LEGEND.to_owned(), + }, Style::default().fg(theme.muted), ))), body[3], @@ -414,7 +451,9 @@ pub fn draw_services_manager( } // Approval modal sits on top of the list. - if let Some(pending) = &sm.approval { + if let Some(pending) = &sm.approval + && rows.iter().any(|row| row.id == pending.service_id) + { draw_approval(f, area, &pending.request, pending.choice, theme); } } @@ -437,6 +476,7 @@ mod tests { fn inst(id: &str, model: &str, tps: Option) -> Instance { Instance { + managed: true, container_id: id.into(), container_name: id.into(), model_name: model.into(), @@ -454,6 +494,197 @@ mod tests { m } + #[test] + fn external_only_lifecycle_keys_do_not_offer_approval_or_spawn_jobs() { + let mut external = inst("svc-external", "external-model", Some(12.0)); + external.managed = false; + let insts = HashMap::from([(external.container_id.clone(), external)]); + for action in ['s', 'r'] { + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + for pressed in [action, 'y'] { + let fx = on_key( + &mut services, + &mut jobs, + &insts, + key(KeyCode::Char(pressed)), + ); + assert!(fx.is_empty(), "external service must never emit effects"); + let sm = services.as_ref().unwrap(); + assert!(sm.approval.is_none(), "external service is read-only"); + assert!(sm.active_job.is_none()); + assert!(jobs.jobs.is_empty()); + } + let out = render(services.as_ref().unwrap(), &jobs, &insts); + assert!(out.contains("No managed services."), "{out}"); + assert!(!out.contains("external-model"), "{out}"); + } + } + + #[test] + fn rows_use_positive_ownership_not_id_or_engine_markers() { + let mut insts = instances(); + let mut external = inst("svc-looks-managed", "external", None); + external.managed = false; + insts.insert("external".into(), external); + let mut managed = inst("lemonade-owned", "managed", None); + managed.container_name = "lemonade".into(); + insts.insert("managed".into(), managed); + let rows = service_rows(&insts); + assert_eq!( + rows.iter().map(|r| r.id.as_str()).collect::>(), + vec!["lemonade-owned", "svc-a", "svc-b"] + ); + let out = render(&ServicesManagerState::default(), &State::default(), &insts); + assert!(out.contains("lemonade-owned"), "{out}"); + assert!(!out.contains("svc-looks-managed"), "{out}"); + } + + #[test] + fn removed_or_unmanaged_target_invalidates_pending_approval() { + for action in ['s', 'r'] { + for replace_with_external in [false, true] { + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + let mut telemetry = State::default(); + let managed = inst("svc-a", "managed", None); + telemetry.apply(StateEvent::InstanceUpserted(managed.clone())); + on_key( + &mut services, + &mut jobs, + &telemetry.instances, + key(KeyCode::Char(action)), + ); + assert!(services.as_ref().unwrap().approval.is_some()); + telemetry.apply(StateEvent::InstanceRemoved("svc-a".into())); + if replace_with_external { + let mut external = managed; + external.managed = false; + telemetry.apply(StateEvent::InstanceUpserted(external)); + } + let out = render(services.as_ref().unwrap(), &jobs, &telemetry.instances); + assert!(out.contains("No managed services."), "{out}"); + assert!( + !out.contains("Review:"), + "stale approval must not render: {out}" + ); + let fx = on_key( + &mut services, + &mut jobs, + &telemetry.instances, + key(KeyCode::Char('y')), + ); + assert!(fx.is_empty(), "stale approval must not emit effects"); + assert!(services.as_ref().unwrap().approval.is_none()); + assert!(services.as_ref().unwrap().active_job.is_none()); + assert!(jobs.jobs.is_empty()); + } + } + } + + #[test] + fn stale_approval_does_not_swallow_the_next_key() { + // `j` after the stale approval is dropped moves the selection. + let mut insts = instances(); + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('s'))); + assert_eq!( + services + .as_ref() + .unwrap() + .approval + .as_ref() + .unwrap() + .service_id, + "svc-a" + ); + // The target disappears while the modal is open. + insts.remove("a"); + insts.insert("c".into(), inst("svc-c", "other", None)); + insts.insert("d".into(), inst("svc-d", "other", None)); + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('j'))); + assert!(fx.is_empty()); + let sm = services.as_ref().unwrap(); + assert!(sm.approval.is_none()); + assert_eq!(sm.selected, 1, "j must be handled as navigation"); + + // `s` after a stale approval stages a fresh approval for the selection. + let mut services = Some(ServicesManagerState::default()); + let mut insts = instances(); + on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('s'))); + insts.remove("a"); + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('s'))); + assert!(fx.is_empty()); + let pending = services.as_ref().unwrap().approval.as_ref().unwrap(); + assert_eq!(pending.service_id, "svc-b"); + // `y` still approves only the fresh target, never the stale one. + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('y'))); + assert!(!fx.is_empty()); + assert_eq!(jobs.jobs.len(), 1); + assert!( + jobs.jobs.contains_key("svc-stop-svc-b"), + "{:?}", + jobs.jobs.keys() + ); + } + + #[test] + fn stale_approval_then_y_cannot_spawn_a_job() { + let mut insts = instances(); + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('s'))); + insts.clear(); + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('y'))); + assert!(fx.is_empty()); + assert!(jobs.jobs.is_empty()); + assert!(services.as_ref().unwrap().approval.is_none()); + } + + #[test] + fn selection_is_clamped_when_managed_rows_shrink() { + let mut insts = instances(); + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('j'))); + assert_eq!(services.as_ref().unwrap().selected, 1); + // Only one managed row remains; the old selection is out of range. + insts.remove("b"); + on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('s'))); + let sm = services.as_ref().unwrap(); + assert_eq!(sm.selected, 0); + let pending = sm.approval.as_ref().expect("a valid target is staged"); + assert_eq!(pending.service_id, "svc-a"); + } + + #[test] + fn restart_requires_approval_then_spawns_job() { + let mut services = Some(ServicesManagerState::default()); + let mut jobs = State::default(); + let insts = instances(); + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('r'))); + assert!(fx.is_empty()); + assert!(jobs.jobs.is_empty()); + let sm = services.as_ref().unwrap(); + assert_eq!( + sm.approval.as_ref().unwrap().action, + LifecycleAction::Restart + ); + let out = render(sm, &jobs, &insts); + assert!(out.contains("restart service"), "{out}"); + let fx = on_key(&mut services, &mut jobs, &insts, key(KeyCode::Char('y'))); + assert_eq!(fx.len(), 1); + assert!(matches!(&fx[0], SideEffect::SpawnJob { args, .. } + if args == &["services", "restart", "svc-a", "--yes"])); + assert!(services.as_ref().unwrap().approval.is_none()); + assert_eq!( + services.as_ref().unwrap().active_job.as_deref(), + Some("svc-restart-svc-a") + ); + assert_eq!(jobs.jobs.len(), 1); + } + #[test] fn rows_are_sorted_by_id() { let rows = service_rows(&instances()); @@ -656,6 +887,63 @@ mod tests { assert!(out.contains("s stop"), "lifecycle hints"); } + #[test] + fn snapshot_labels_a_last_request_rate() { + let mut insts = HashMap::new(); + for (id, last_request) in [("live", false), ("idle", true)] { + let mut i = inst(id, &format!("model-{id}"), Some(42.0)); + i.gen_tps_last_request = last_request; + insts.insert(id.into(), i); + } + let rows = service_rows(&insts); + assert!(rows[0].gen_tps_last_request && !rows[1].gen_tps_last_request); + let out = render(&ServicesManagerState::default(), &State::default(), &insts); + assert!(out.contains("req 42"), "{out}"); + assert!(out.contains("gen 42"), "{out}"); + assert!(out.contains(format::LAST_REQ_LEGEND), "{out}"); + } + + /// The rate shares `gen`'s column budget, so at the widths where the `gen` + /// value fits, the last-request value fits too (it used to be clipped to a + /// different number, `last req 1` for 137.5). + #[test] + fn a_last_request_rate_is_not_clipped_where_a_live_rate_fits() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + let theme = Theme::from_name("default-dark"); + for last_request in [false, true] { + let mut i = inst("svc", "model-svc", Some(137.5)); + i.gen_tps_last_request = last_request; + let insts = HashMap::from([("svc".to_string(), i)]); + let mut term = Terminal::new(TestBackend::new(98, 28)).unwrap(); + term.draw(|f| { + draw_services_manager( + f, + f.area(), + &ServicesManagerState::default(), + &insts, + &State::default(), + 0, + &theme, + ); + }) + .unwrap(); + let out: String = term + .backend() + .buffer() + .content() + .iter() + .map(ratatui::buffer::Cell::symbol) + .collect(); + let label = format::gen_rate_label_short(last_request); + assert!( + out.contains(&format!("{label} 137.5")), + "last_request={last_request}: +{out}" + ); + } + } + #[test] fn snapshot_shows_held_legend_when_a_row_is_held() { use rocm_dash_core::metrics::{InstanceStatus, ObservationFreshness, ObservationMetadata}; @@ -663,6 +951,7 @@ mod tests { insts.insert( "held".into(), Instance { + managed: true, container_id: "held".into(), container_name: "svc-held".into(), model_name: "llama3".into(), @@ -718,6 +1007,7 @@ mod tests { m.insert( id.clone(), Instance { + managed: true, container_id: id.clone(), container_name: id, model_name: "llama3".into(), diff --git a/crates/rocm-dash-tui/src/ui/tabs/home.rs b/crates/rocm-dash-tui/src/ui/tabs/home.rs index f257f3f1cd..d7ff77ddce 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/home.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/home.rs @@ -19,6 +19,7 @@ use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; use ratatui::widgets::Paragraph; +use rocm_dash_core::metrics::Instance; use rocm_dash_core::state::JobStatus; use crate::app::{AppState, ConnState, UpdateStatus}; @@ -377,9 +378,11 @@ fn draw_hero_left(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { // Tokens/watt: summed across running instances when available. // Mark held if any contributing instance has a held gen_tps observation // (tok/W derives from gen_tps; aggregate inherits held status). - let tpw = instance_aggregate(state, |i| i.tokens_per_watt); + // `live_tokens_per_watt` leaves out an efficiency derived from a Lemonade + // last-request rate, for the same reason as the T/S hero. + let tpw = instance_aggregate(state, Instance::live_tokens_per_watt); let any_tpw_held = state.instances.values().any(|i| { - i.tokens_per_watt.is_some_and(f64::is_finite) + i.live_tokens_per_watt().is_some_and(f64::is_finite) && i.gen_tps_observation .as_ref() .is_some_and(|m| m.freshness == rocm_dash_core::metrics::ObservationFreshness::Held) @@ -1300,6 +1303,63 @@ mod tests { ); } + /// The text of the screen row holding `needle`, for a `cols`-wide render. + fn row_with(out: &str, cols: usize, needle: &str) -> String { + let at = out + .find(needle) + .unwrap_or_else(|| panic!("{needle:?} not rendered:\n{out}")); + let chars: Vec = out.chars().collect(); + let row = out[..at].chars().count() / cols; + chars[row * cols..(row + 1) * cols].iter().collect() + } + + /// An idle Lemonade server keeps its last request's rate. The same numbers on + /// a live instance fill the T/S and tokens/watt heroes; as a last-request rate + /// they must not, and held metadata on it adds no marker or legend. + #[test] + fn home_heroes_leave_out_a_last_request_rate() { + let live = render( + &state_with_instance(instance_with_obs(300.0, Some(fresh_obs()))), + 160, + 30, + ); + assert!(row_with(&live, 160, "T/S").contains("300"), "{live}"); + assert!(live.contains("1.0 tokens / watt"), "{live}"); + + for obs in [fresh_obs(), held_obs()] { + let mut inst = instance_with_obs(300.0, Some(obs)); + inst.gen_tps_last_request = true; + let out = render(&state_with_instance(inst), 160, 30); + let tps_row = row_with(&out, 160, "T/S"); + assert!(!tps_row.contains("300"), "{tps_row}"); + assert!(out.contains("tokens / watt —"), "{out}"); + assert!(!out.contains("1.0 tokens / watt"), "{out}"); + assert!(!out.contains(format::HELD_LEGEND), "{out}"); + } + } + + /// Beside a live instance the totals are non-zero, so whether they are marked + /// held is decided by the held check alone: a held last-request rate is left + /// out of the totals and must not mark them held either. + #[test] + fn a_held_last_request_rate_does_not_mark_live_totals_held() { + let mut s = state_with_instance(instance_with_obs(300.0, Some(fresh_obs()))); + let mut last = instance_with_obs(150.0, Some(held_obs())); + last.container_id = "last".into(); + last.gen_tps_last_request = true; + s.instances.insert(last.container_id.clone(), last); + let out = render(&s, 160, 30); + let tps_row = row_with(&out, 160, "T/S"); + assert!(tps_row.contains("300"), "{tps_row}"); + assert!(!tps_row.contains(format::HELD_MARKER), "{tps_row}"); + assert!(out.contains("1.0 tokens / watt"), "{out}"); + assert!( + !out.contains(&format!("tokens / watt{}", format::HELD_MARKER)), + "{out}" + ); + assert!(!out.contains(format::HELD_LEGEND), "{out}"); + } + #[test] fn hero_never_shows_a_held_marker_without_its_legend() { // Squeezing the hero drops the legend row before the value rows, so diff --git a/crates/rocm-dash-tui/src/ui/tabs/instances.rs b/crates/rocm-dash-tui/src/ui/tabs/instances.rs index 62ebdaf811..282e326f25 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/instances.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/instances.rs @@ -10,7 +10,7 @@ use ratatui::Frame; use ratatui::layout::{Constraint, Direction, Layout, Rect}; use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; -use ratatui::widgets::{Cell, Paragraph, Row, Table}; +use ratatui::widgets::{Cell, Paragraph, Row, Table, Wrap}; use rocm_dash_core::metrics::{ Instance, InstanceStatus, ObservationFreshness, ObservationMetadata, @@ -90,7 +90,11 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { || (is_held(&inst.ttft_observation) && inst.ttft_ms.is_some_and(f64::is_finite)) || (is_held(&inst.tpot_observation) && inst.tpot_ms.is_some_and(f64::is_finite)) }); - let (table_area, legend_area) = if any_held { + let any_last_req = instances + .iter() + .take(visible_rows) + .any(|inst| inst.gen_tps_last_request && inst.gen_tps.is_some_and(f64::is_finite)); + let (table_area, legend_area) = if any_held || any_last_req { let split = Layout::default() .direction(Direction::Vertical) .constraints([Constraint::Min(0), Constraint::Length(1)]) @@ -113,8 +117,12 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { let rows = instances.iter().enumerate().map(|(i, inst)| { let model = trunc(&inst.model_name, 22); let tps = format::gen_tps_cell(inst.gen_tps, inst.gen_tps_observation.as_ref()); - let tpw = - format::tokens_per_watt_cell(inst.tokens_per_watt, inst.gen_tps_observation.as_ref()); + // A last-request rate has no efficiency to show: the figure beside it + // is idle board power over a finished request, not a current rate. + let tpw = format::tokens_per_watt_cell( + inst.live_tokens_per_watt(), + inst.gen_tps_observation.as_ref(), + ); let ttft = format::latency_cell(inst.ttft_ms, inst.ttft_observation.as_ref()); let tpot = format::latency_cell(inst.tpot_ms, inst.tpot_observation.as_ref()); let power = inst @@ -171,7 +179,13 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { if let Some(legend_area) = legend_area { f.render_widget( Paragraph::new(Line::from(Span::styled( - format::HELD_LEGEND, + match (any_held, any_last_req) { + (true, true) => { + format!("{} · {}", format::HELD_LEGEND, format::LAST_REQ_LEGEND) + } + (true, false) => format::HELD_LEGEND.to_owned(), + _ => format::LAST_REQ_LEGEND.to_owned(), + }, Style::default().fg(theme.muted), ))), legend_area, @@ -179,13 +193,16 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { } } -/// Sort instances deterministically by container_name so that an index from -/// AppState always maps to the same row in the table. +/// Sort instances deterministically by container_name, then container_id, so +/// that an index from AppState always maps to the same row in the table. The id +/// matters: every external Lemonade model shares the name `lemonade`. pub(in crate::ui) fn sorted_instances( instances: &std::collections::HashMap, ) -> Vec<&Instance> { let mut v: Vec<&Instance> = instances.values().collect(); - v.sort_by(|a, b| a.container_name.cmp(&b.container_name)); + v.sort_by(|a, b| { + (&a.container_name, &a.container_id).cmp(&(&b.container_name, &b.container_id)) + }); v } @@ -210,16 +227,18 @@ fn draw_empty(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { } else { match &state.conn { ConnState::Connected { .. } => { - "no instances · start daemon with --enable-docker on a host with vLLM containers" + "no serving models discovered · open Serving to launch a managed model or check an external endpoint" } _ => "waiting for daemon…", } }; let inner = panel::bento(f, area, Some("Instances"), BoxRole::Neutral, false, theme); + // Wrapped, not clipped: the second half names the external-endpoint path. let p = Paragraph::new(Line::from(Span::styled( body, Style::default().fg(theme.muted), - ))); + ))) + .wrap(Wrap { trim: false }); f.render_widget(p, inner); } @@ -230,6 +249,7 @@ pub(in crate::ui) mod tests { pub(in crate::ui) fn mk_inst(name: &str) -> Instance { Instance { + managed: false, container_id: format!("id-{name}"), container_name: name.to_string(), status: InstanceStatus::Running, @@ -246,6 +266,7 @@ pub(in crate::ui) mod tests { waiting_reqs: None, gen_tps: None, gen_tps_observation: None, + gen_tps_last_request: false, tokens_per_watt: None, power_w: None, ttft_ms: None, @@ -339,6 +360,28 @@ pub(in crate::ui) mod tests { ); } + /// Every external Lemonade model on an endpoint shares the container name + /// `lemonade`, so the name alone leaves their order to the map's hash order, + /// and the selection index would land on a different model from one frame + /// to the next. The id breaks the tie. + #[test] + fn sorted_instances_breaks_a_name_tie_by_id() { + for _ in 0..32 { + // A fresh map each pass: each one gets its own hash seed. + let mut m = HashMap::new(); + for id in ["lemonade-m-c", "lemonade-m-a", "lemonade-m-b"] { + let mut inst = mk_inst("lemonade"); + inst.container_id = id.into(); + m.insert(inst.container_id.clone(), inst); + } + let ids: Vec<&str> = sorted_instances(&m) + .iter() + .map(|i| i.container_id.as_str()) + .collect(); + assert_eq!(ids, ["lemonade-m-a", "lemonade-m-b", "lemonade-m-c"]); + } + } + #[test] fn selected_instance_lookup_by_sorted_index() { // HashMap insertion order is irrelevant — sorted_instances orders by name. @@ -357,6 +400,7 @@ pub(in crate::ui) mod tests { services_past_attempts: 0, managed_registry_probe: None, launcher_hosted: false, + lemonade_managed_installed: None, connect: "test".into(), conn: ConnState::Initial, latest: None, @@ -978,4 +1022,127 @@ pub(in crate::ui) mod tests { "zero gen_tps must render as '0', not '—'; got:\n{out}" ); } + + pub(in crate::ui) fn last_req_inst(name: &str, last_request: bool) -> Instance { + let mut inst = mk_inst_obs(name, Some(37.5), Some(obs(ObservationFreshness::Fresh, 0))); + inst.gen_tps_last_request = last_request; + inst + } + + #[test] + fn table_labels_a_last_request_rate_and_only_then() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + for (last_request, expect_legend) in [(true, true), (false, false)] { + let state = state_with_snap(last_req_inst("rate", last_request)); + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!(out.contains("38"), "the value itself still shows:\n{out}"); + assert_eq!( + out.contains(format::LAST_REQ_LEGEND), + expect_legend, + "last_request={last_request}:\n{out}" + ); + } + } + + /// The legend explains a rate that is on screen: a last-request flag on a row + /// with no finite rate has nothing to explain. + #[test] + fn table_has_no_last_request_legend_without_a_rate_to_explain() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + for gen_tps in [None, Some(f64::NAN)] { + let mut inst = last_req_inst("rate", true); + inst.gen_tps = gen_tps; + let state = state_with_snap(inst); + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!( + !out.contains(format::LAST_REQ_LEGEND), + "gen_tps={gen_tps:?}: +{out}" + ); + } + } + + /// A held rate that is also a last-request rate shows both legends, joined. + #[test] + fn table_shows_the_held_and_last_request_legends_together() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + let mut inst = mk_inst_obs( + "both", + Some(37.5), + Some(obs(ObservationFreshness::Held, 30)), + ); + inst.gen_tps_last_request = true; + let state = state_with_snap(inst); + // 80 columns is the width that matters: the joined legend must fit. + let mut term = Terminal::new(TestBackend::new(80, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + let both = format!("{} · {}", format::HELD_LEGEND, format::LAST_REQ_LEGEND); + assert!(out.contains(&both), "{out}"); + } + + /// The legend wording is what the user reads (and what the e2e screen + /// scenario looks for): it must name Lemonade and the last-request rate. + #[test] + fn the_last_request_legend_names_what_it_explains() { + assert_eq!( + format::LAST_REQ_LEGEND, + "Lemonade: last req tok/s, not live" + ); + } + + /// An efficiency derived from a last-request rate is not shown on the row, + /// whether or not the rate itself is (replayed recordings can carry one). + #[test] + fn table_leaves_out_an_efficiency_derived_from_a_last_request_rate() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + for gen_tps in [Some(37.5), None] { + let mut inst = last_req_inst("rate", true); + inst.gen_tps = gen_tps; + inst.tokens_per_watt = Some(0.12); + let state = state_with_snap(inst); + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!( + !out.contains("0.12"), + "gen_tps={gen_tps:?}: +{out}" + ); + } + } + + #[test] + fn the_connected_empty_state_names_the_external_endpoint_path_at_80_columns() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + let mut state = mk_state(HashMap::new(), 0); + state.conn = ConnState::Connected { + host: "localhost".into(), + version: "0.0.0".into(), + }; + let mut term = Terminal::new(TestBackend::new(80, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let grid = buffer_text(&term); + assert!(grid.contains("external"), "{grid}"); + assert!(grid.contains("endpoint"), "{grid}"); + } } diff --git a/crates/rocm-dash-tui/src/ui/tabs/observe.rs b/crates/rocm-dash-tui/src/ui/tabs/observe.rs index fbef42510a..5d82108948 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/observe.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/observe.rs @@ -461,6 +461,55 @@ mod tests { } } + fn last_request_instance( + model: &str, + gen_tps: f64, + obs: Option, + ) -> Instance { + Instance { + gen_tps_last_request: true, + ..instance_with_obs(model, gen_tps, obs) + } + } + + /// Only the two hero panels, so the Instances table (which labels a + /// last-request rate and keeps its own per-row tok/W) cannot satisfy or break + /// an assertion about them. + fn render_heroes(state: &AppState) -> String { + let mut term = Terminal::new(TestBackend::new(160, 12)).unwrap(); + term.draw(|f| draw_hero_band(f, f.area(), state, &state.theme)) + .unwrap(); + term.backend() + .buffer() + .content() + .iter() + .map(ratatui::buffer::Cell::symbol) + .collect() + } + + /// An idle Lemonade server keeps reporting its last request's rate. The Node + /// efficiency and Node throughput heroes must not present it as current: the + /// same rate as a live instance fills them (`1.00 tok/W`, a `tok/s` figure), as a + /// last-request rate they stay at `—`, and held metadata on it adds no marker + /// or legend. + #[test] + fn observe_heroes_leave_out_a_last_request_rate() { + use crate::ui::format; + let live = connected_with_instances(vec![instance_with_obs("m", 300.0, Some(fresh_obs()))]); + let out = render_heroes(&live); + assert!(out.contains("1.00 tok/W"), "baseline: {out}"); + assert!(out.contains(" tok/s"), "baseline: {out}"); + + for obs in [fresh_obs(), held_obs()] { + let last = connected_with_instances(vec![last_request_instance("m", 300.0, Some(obs))]); + let out = render_heroes(&last); + assert!(!out.contains("1.00 tok/W"), "{out}"); + assert!(!out.contains(" tok/s"), "{out}"); + assert!(!out.contains(format::HELD_LEGEND), "{out}"); + assert!(!out.contains(format::HELD_MARKER), "{out}"); + } + } + // ── EAI-7960: HELD_LEGEND rendering and efficiency hero held marker ─────── #[test] diff --git a/crates/rocm-dash-tui/src/ui/tabs/pane.rs b/crates/rocm-dash-tui/src/ui/tabs/pane.rs index be6373134c..30b6babc42 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/pane.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/pane.rs @@ -389,23 +389,56 @@ fn live_lines(action: KeyAction, state: &AppState, theme: &Theme) -> (Vec { let mut lines = vec![head("Engines".into())]; - let lemon = info().and_then(|i| i.lemond_version.clone()); + // External endpoint evidence is live; managed installation evidence + // is a separate launch-time Detect snapshot. GPU metadata carries + // neither verdict and must never imply an absent engine. + let external_running = state + .instances + .values() + .any(|i| i.is_external_lemonade() && i.status.is_serving()); + let lemon = if external_running { + "Lemonade — externally running" + } else { + match state.lemonade_managed_installed { + Some(true) => "Lemonade — managed runtime installed", + Some(false) => "Lemonade — managed runtime not installed", + None => "Lemonade — unknown, run `rocm engines list`", + } + }; lines.push(Line::from(vec![ - mark(lemon.is_some()), - Span::styled( - lemon.map_or_else( - || "Lemonade — not installed".into(), - |v| format!("Lemonade {v}"), - ), - Style::default().fg(theme.fg), - ), + mark(external_running || state.lemonade_managed_installed == Some(true)), + Span::styled(lemon, Style::default().fg(theme.fg)), ])); - // The daemon doesn't surface vLLM detection yet — open the manager to - // check rather than claiming a status we don't have. + if external_running { + let managed = match state.lemonade_managed_installed { + Some(true) => "Managed runtime — installed", + Some(false) => "Managed runtime — not installed", + None => "Managed runtime — unknown, run `rocm engines list`", + }; + lines.push(Line::from(Span::styled( + managed, + Style::default().fg(theme.muted), + ))); + } + // The daemon doesn't surface vLLM detection yet. `rocm engines list` + // prints each engine's Detect result, so point there rather than + // claiming a status we don't have. lines.push(Line::from(vec![ Span::styled("· ", Style::default().fg(theme.muted)), - Span::styled("vLLM — open to check", Style::default().fg(theme.muted)), + Span::styled( + "vLLM — unknown, run `rocm engines list`", + Style::default().fg(theme.muted), + ), ])); + // The managed-runtime verdict is a launch-time snapshot: installing + // from the Engines overlay does not update it. The test + // `managed_install_verdict_is_not_changed_by_live_events` proves it. + if state.lemonade_managed_installed.is_some() { + lines.push(Line::from(Span::styled( + "Install status is read at startup. Restart the dashboard to refresh.", + Style::default().fg(theme.muted), + ))); + } (lines, false) } KeyAction::OpenServeWizard => { @@ -430,13 +463,18 @@ fn live_lines(action: KeyAction, state: &AppState, theme: &Theme) -> (Vec { // Starting services are listed (with their stage) so a fresh serve - // is visible before it passes its readiness probe. - let running = format::running_instances(state); - let mut lines = vec![head(format!("Running now — {}", running.len()))]; + // is visible before it passes its readiness probe. Only services this + // host manages are listed: the Services overlay this pane opens has no + // row for an external or discovered endpoint. + let running: Vec<_> = format::running_instances(state) + .into_iter() + .filter(|i| crate::ui::services_manager::is_listed_service(i)) + .collect(); + let mut lines = vec![head(format!("Managed running — {}", running.len()))]; let mut any_held = false; if running.is_empty() { lines.push(Line::from(Span::styled( - " none running", + " no managed services running", Style::default().fg(theme.muted), ))); } else { @@ -469,7 +507,8 @@ fn live_lines(action: KeyAction, state: &AppState, theme: &Theme) -> (Vec (Vec (Vec::new(), false), } } + +#[cfg(test)] +mod tests { + use super::*; + + fn engine_text(state: &AppState) -> String { + live_lines(KeyAction::OpenEngineManager, state, &Theme::default()) + .0 + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") + } + + fn glyph_of(state: &AppState, needle: &str) -> String { + let (lines, _) = live_lines(KeyAction::OpenEngineManager, state, &Theme::default()); + let line = lines + .iter() + .find(|l| l.to_string().contains(needle)) + .unwrap_or_else(|| panic!("no line containing {needle:?}")); + line.spans[0].content.to_string() + } + + #[test] + fn lemonade_line_glyph_is_check_only_when_installed() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.lemonade_managed_installed = Some(true); + assert_eq!( + glyph_of(&state, "Lemonade — managed runtime installed"), + "✓ " + ); + state.lemonade_managed_installed = Some(false); + assert_eq!( + glyph_of(&state, "Lemonade — managed runtime not installed"), + "· " + ); + state.lemonade_managed_installed = None; + assert_eq!(glyph_of(&state, "Lemonade — unknown"), "· "); + } + + #[test] + fn external_running_lemonade_line_has_check_glyph() { + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.instances.insert( + "x".into(), + Instance { + managed: false, + container_id: "x".into(), + container_name: "lemonade".into(), + status: InstanceStatus::Ready, + ..Instance::default() + }, + ); + assert_eq!(glyph_of(&state, "Lemonade — externally running"), "✓ "); + } + + #[test] + fn startup_snapshot_note_shown_only_with_a_verdict() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + assert!(!engine_text(&state).contains("read at startup")); + state.lemonade_managed_installed = Some(false); + let text = engine_text(&state); + // This pins the wording only. That the status really is captured once + // at startup is proved by + // `managed_install_verdict_is_not_changed_by_live_events`. + assert!( + text.contains("Install status is read at startup. Restart the dashboard to refresh."), + "{text}" + ); + assert!( + text.contains("vLLM — unknown, run `rocm engines list`"), + "{text}" + ); + } + + #[test] + fn managed_install_verdict_is_not_changed_by_live_events() { + // Proves the claim the pane's "read at startup" note makes: the verdict + // is set once from the startup args (`event_loop.rs`) and no later + // daemon event rewrites it, so the note is true. + use rocm_dash_core::metrics::{Instance, InstanceStatus, Snapshot}; + use rocm_dash_core::protocol::Event; + + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.lemonade_managed_installed = Some(false); + let managed = Instance { + managed: true, + container_id: "svc".into(), + container_name: "lemonade".into(), + status: InstanceStatus::Ready, + ..Instance::default() + }; + state.apply_event(Event::InstanceDiscovered(managed)); + state.apply_event(Event::Snapshot(Snapshot::default())); + state.apply_event(Event::InstanceGone { + container_id: "svc".into(), + }); + assert_eq!(state.lemonade_managed_installed, Some(false)); + assert!(engine_text(&state).contains("Lemonade — managed runtime not installed")); + } + + #[test] + fn installed_stopped_managed_lemonade_is_installed_without_gpu_metadata() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.lemonade_managed_installed = Some(true); + assert!(state.instances.is_empty()); + assert!(engine_text(&state).contains("Lemonade — managed runtime installed")); + assert!(!engine_text(&state).contains("running")); + } + + #[test] + fn no_instances_and_no_managed_runtime_reports_lemonade_not_installed() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.lemonade_managed_installed = Some(false); + assert!(engine_text(&state).contains("Lemonade — managed runtime not installed")); + } + + #[test] + fn external_running_and_stopped_instances_do_not_invent_managed_installation() { + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + let id = "lemonade-localhost-13305-model-61".to_owned(); + state.instances.insert( + id.clone(), + Instance { + container_id: id.clone(), + container_name: "lemonade".into(), + status: InstanceStatus::Ready, + ..Instance::default() + }, + ); + let text = engine_text(&state); + assert!(text.contains("Lemonade — externally running")); + assert!(text.contains("Managed runtime — unknown, run `rocm engines list`")); + state.lemonade_managed_installed = Some(true); + assert!(engine_text(&state).contains("Managed runtime — installed")); + state.instances.get_mut(&id).unwrap().status = InstanceStatus::Stopped; + assert!(engine_text(&state).contains("Lemonade — managed runtime installed")); + assert!(!engine_text(&state).contains("externally running")); + } + + #[test] + fn external_lemonade_verdict_uses_ownership_not_id_prefix() { + use rocm_dash_core::metrics::{Instance, InstanceStatus}; + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + let id = "svc-ordinary-id".to_owned(); + state.instances.insert( + id.clone(), + Instance { + managed: false, + container_id: id.clone(), + container_name: "lemonade".into(), + status: InstanceStatus::Ready, + ..Instance::default() + }, + ); + assert!(engine_text(&state).contains("Lemonade — externally running")); + let external = state.instances.remove(&id).unwrap(); + assert!(!engine_text(&state).contains("externally running")); + state.instances.insert( + "lemonade-looks-external".into(), + Instance { + managed: true, + container_id: "lemonade-looks-external".into(), + ..external + }, + ); + assert!(!engine_text(&state).contains("externally running")); + } + + fn services_text(state: &AppState) -> String { + live_lines(KeyAction::OpenServices, state, &Theme::default()) + .0 + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") + } + + fn serving(id: &str, managed: bool) -> rocm_dash_core::metrics::Instance { + rocm_dash_core::metrics::Instance { + managed, + container_id: id.into(), + model_name: format!("model-{id}"), + status: rocm_dash_core::metrics::InstanceStatus::Ready, + ..rocm_dash_core::metrics::Instance::default() + } + } + + #[test] + fn services_preview_ignores_external_only_instances() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + for id in ["ext-a", "ext-b"] { + state.instances.insert(id.into(), serving(id, false)); + } + let text = services_text(&state); + assert!(text.contains("Managed running — 0"), "{text}"); + assert!(text.contains("no managed services running"), "{text}"); + assert!(!text.contains("model-ext"), "{text}"); + // Same set the overlay would list. + assert!(crate::ui::services_manager::service_rows(&state.instances).is_empty()); + } + + #[test] + fn services_preview_labels_a_last_request_rate() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + for (id, last_request) in [("live", false), ("idle", true)] { + let mut inst = serving(id, true); + inst.gen_tps = Some(42.0); + inst.gen_tps_last_request = last_request; + state.instances.insert(id.into(), inst); + } + let text = services_text(&state); + let line_of = |model: &str| { + text.lines() + .find(|l| l.contains(model)) + .unwrap_or_else(|| panic!("{model} missing in {text}")) + .to_owned() + }; + assert!(line_of("model-idle").contains("last req 42"), "{text}"); + assert!(line_of("model-live").contains("gen 42"), "{text}"); + } + + #[test] + fn services_preview_counts_only_managed_instances() { + let mut state = AppState::new("unix:/unused".into(), "dark".into()); + state.instances.insert("ext".into(), serving("ext", false)); + state.instances.insert("own".into(), serving("own", true)); + let text = services_text(&state); + assert!(text.contains("Managed running — 1"), "{text}"); + assert!(text.contains("model-own"), "{text}"); + assert!(!text.contains("model-ext"), "{text}"); + } + + #[test] + fn unknown_lemonade_installation_is_not_claimed_absent() { + let state = AppState::new("unix:/unused".into(), "dark".into()); + let (lines, _) = live_lines(KeyAction::OpenEngineManager, &state, &Theme::default()); + let text = lines + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n"); + assert!( + text.contains("Lemonade — unknown, run `rocm engines list`"), + "{text}" + ); + assert!( + !text.contains("not installed"), + "missing GPU metadata is not installation evidence" + ); + } +} diff --git a/crates/rocm-dash-tui/src/ui/tabs/serving.rs b/crates/rocm-dash-tui/src/ui/tabs/serving.rs index ae63e93bf3..74f226a234 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/serving.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/serving.rs @@ -210,6 +210,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Running, port: Some(8000), @@ -219,7 +220,7 @@ mod tests { let out = render(&s, 120, 28); // The running instance is shown inline in the detail pane — no popup. assert!( - out.contains("Running now"), + out.contains("Managed running"), "live running header missing: {out:?}" ); assert!( @@ -242,6 +243,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Ready, port: Some(8000), @@ -250,7 +252,7 @@ mod tests { ); let out = render(&s, 120, 28); assert!( - out.contains("Running now — 1"), + out.contains("Managed running — 1"), "ready instance not counted as running: {out:?}" ); assert!( @@ -267,6 +269,7 @@ mod tests { s.serving_sel = 2; let mk = |name: &str, status| Instance { container_name: name.into(), + managed: true, model_name: name.into(), status, ..Default::default() @@ -283,7 +286,7 @@ mod tests { s.instances .insert("b".into(), mk("Dead-Model", InstanceStatus::Stopped)); let out = render(&s, 120, 28); - assert!(out.contains("Running now — 1"), "{out:?}"); + assert!(out.contains("Managed running — 1"), "{out:?}"); assert!(out.contains("Starting-Model"), "{out:?}"); assert!(out.contains("downloading"), "{out:?}"); assert!(!out.contains("none running"), "{out:?}"); @@ -296,6 +299,7 @@ mod tests { ) -> rocm_dash_core::metrics::Instance { rocm_dash_core::metrics::Instance { container_name: name.into(), + managed: true, model_name: name.into(), status: rocm_dash_core::metrics::InstanceStatus::Starting { phase }, ..Default::default() @@ -315,7 +319,7 @@ mod tests { s.instances .insert("a".into(), starting_inst("Docker-Model", None)); let out = render(&s, 120, 28); - assert!(out.contains("Running now — 1"), "{out:?}"); + assert!(out.contains("Managed running — 1"), "{out:?}"); assert!( out.contains("docker-model") || out.contains("Docker-Model"), "{out:?}" @@ -346,6 +350,7 @@ mod tests { "b".into(), Instance { container_name: "Z-Serving".into(), + managed: true, model_name: "Z-Serving".into(), status: InstanceStatus::Ready, ..Default::default() @@ -365,7 +370,7 @@ mod tests { .insert(format!("s{n}"), starting_inst(&format!("Start-{n}"), None)); } let out = render(&s, 120, 40); - assert!(out.contains("Running now — 7"), "{out:?}"); + assert!(out.contains("Managed running — 7"), "{out:?}"); assert!(out.contains("…and 2 more"), "{out:?}"); } @@ -399,6 +404,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Running, port: Some(8000), @@ -437,6 +443,7 @@ mod tests { format!("vllm-{n}"), Instance { container_name: format!("vllm-{n}"), + managed: true, model_name: format!("Model-{n}"), status: InstanceStatus::Running, port: Some(8000 + n), @@ -450,7 +457,7 @@ mod tests { ); } // area.height=8 → bento inner.height=5 for the detail column, tall - // enough for summary + blank + "Running now" header + the first + // enough for summary + blank + "Managed running" header + the first // running row, but far short of the old inline legend's position // after 5 rows + the "…and N more" line. let out = render(&s, 160, 8); @@ -472,6 +479,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Running, port: Some(8000), @@ -500,6 +508,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Running, port: Some(8000), @@ -530,6 +539,7 @@ mod tests { "vllm-1".into(), Instance { container_name: "vllm-1".into(), + managed: true, model_name: "Llama-3.1-8B".into(), status: InstanceStatus::Running, port: Some(8000), diff --git a/crates/rocm-dash-tui/tests/dash_characterization.rs b/crates/rocm-dash-tui/tests/dash_characterization.rs index fc7f8e659a..43fa0cf537 100644 --- a/crates/rocm-dash-tui/tests/dash_characterization.rs +++ b/crates/rocm-dash-tui/tests/dash_characterization.rs @@ -181,9 +181,11 @@ fn observe_empty_state_shows_placeholders() { }; s.latest = Some(synthetic_snapshot()); let out = render(&mut s, 160, 44); + // @id:dash-lemonade-external-protocol-transitions proves model absence + // together with this message and navigation to the named Serving view. assert!( - out.contains("no instances"), - "empty instances placeholder: {out:?}" + out.contains("no serving models discovered"), + "empty serving-model placeholder: {out:?}" ); } diff --git a/docs/architecture.md b/docs/architecture.md index a303a91f88..6c754b41b1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -396,7 +396,7 @@ The pure layer under the dashboard: telemetry, benchmark and protocol types, the | [`bench_schema.rs`](../crates/rocm-dash-core/src/bench_schema.rs) | `BenchmarkRow`, the one row type for a benchmark result, and `PassFail`, vendored from the instinct-agent-bench schema. | flat | | [`chat_template.rs`](../crates/rocm-dash-core/src/chat_template.rs) | The chat-template defaults a locally served model needs on an OpenAI-compatible chat request (`local_model_should_disable_thinking`). | flat | | [`config.rs`](../crates/rocm-dash-core/src/config.rs) | The standalone dashboard configuration in `rocm-dash/config.toml` under the config directory (daemon, TUI and per-engine sections) and the default daemon socket path. | flat | -| [`efficiency.rs`](../crates/rocm-dash-core/src/efficiency.rs) | The pure join of throughput and power: GPU-id normalisation, board power attributed to the instances on a GPU by VRAM share (an occupant that is still `Starting` with no VRAM reading, while another occupant has one, is left out of the split so it does not halve a serving neighbour's power), and tokens per watt. `node_throughput` sums the node's generation throughput once and works out its freshness (fresh, held, or unknown), ignoring instances at zero tok/s when judging freshness; the Observe and Home throughput heroes call it directly. `node_power_w` sums board power over the GPUs that report a usable reading, skipping one that is not finite or not positive as `attribute_power_w` does, and is what the Observe, Hardware and Home power figures show and divide by. `node_efficiency` combines the two into the whole-node `NodeEfficiency` (Σ `gen_tps` over that power, including GPUs no instance occupies) and is the one implementation behind the Observe efficiency hero and its trend, the Hardware header, and the assistant's `tokens_per_watt` and `list_instances` tools, which reach `node_throughput` through it (the Home tab's tokens-per-watt figure is a separate sum of per-instance values). | flat | +| [`efficiency.rs`](../crates/rocm-dash-core/src/efficiency.rs) | The pure join of throughput and power: GPU-id normalisation, board power attributed to the instances on a GPU by VRAM share (an occupant that is still `Starting` with no VRAM reading, while another occupant has one, is left out of the split so it does not halve a serving neighbour's power), and tokens per watt. `node_throughput` sums the node's live generation throughput once (a Lemonade endpoint's last-request rate is not live and is left out, as in the per-instance `instance_tokens_per_watt`) and works out its freshness (fresh, held, or unknown), ignoring instances at zero tok/s when judging freshness; the Observe and Home throughput heroes call it directly. `node_power_w` sums board power over the GPUs that report a usable reading, skipping one that is not finite or not positive as `attribute_power_w` does, and is what the Observe, Hardware and Home power figures show and divide by. `node_efficiency` combines the two into the whole-node `NodeEfficiency` (Σ live `gen_tps` over that power, including GPUs no instance occupies) and is the one implementation behind the Observe efficiency hero and its trend, the Hardware header, and the assistant's `tokens_per_watt` and `list_instances` tools, which reach `node_throughput` through it (the Home tab's tokens-per-watt figure is a separate sum of per-instance values). | flat | | [`metrics.rs`](../crates/rocm-dash-core/src/metrics.rs) | The live metric types the dashboard crates share: `Snapshot`, `GpuMetrics`, `Instance` with its `InstanceStatus` predicates and labels, and the observation-freshness metadata. | flat | | [`observation.rs`](../crates/rocm-dash-core/src/observation.rs) | Turns a generation-token counter into an honest rate: a bounded sample window, fresh-or-held reporting with expiry, and directly reported rates, with all time passed in. | flat | | [`partition.rs`](../crates/rocm-dash-core/src/partition.rs) | The GPU compute-partition and memory-partition mode enums. | flat | @@ -434,7 +434,7 @@ The I/O-heavy collector implementations behind `rocm-dash-core`'s traits (amd-sm | [`docker.rs`](../crates/rocm-dash-collectors/src/docker.rs) | `DockerDiscovery`: finds vLLM containers through `bollard` and builds a `DiscoveredService` from each container's environment and command line. | flat | | [`engine_registry.rs`](../crates/rocm-dash-collectors/src/engine_registry.rs) | `EngineKind`: maps a serving engine (vLLM, Lemonade) to its label, conventional port and sample parser. | flat | | [`host.rs`](../crates/rocm-dash-collectors/src/host.rs) | `HostCollector`: CPU, memory and swap from `sysinfo`, sampled once per tick into `SystemMetrics`. Disk and network rates are not collected yet and stay 0. | flat | -| [`lemonade.rs`](../crates/rocm-dash-collectors/src/lemonade.rs) | The Lemonade Server collector: pure parsers over its stats and health endpoints, and discovery of a running server. | flat | +| [`lemonade.rs`](../crates/rocm-dash-collectors/src/lemonade.rs) | The Lemonade Server collector: pure parsers over its stats and health endpoints, and the discovery of a running server's loaded models, one row per model. | flat | | [`parallel.rs`](../crates/rocm-dash-collectors/src/parallel.rs) | `parallel_scrape`: runs an async scrape against many targets concurrently on a `JoinSet`. | flat | | [`vllm_prom.rs`](../crates/rocm-dash-collectors/src/vllm_prom.rs) | The vLLM Prometheus `/metrics` scraper: request gauges, KV-cache usage, the generation-token counter and the latency histogram sums. | flat | | [`winperf.rs`](../crates/rocm-dash-collectors/src/winperf.rs) | Live Windows GPU metrics from OS performance counters, with temperature, power and clocks from ADLX when the AMD driver's DLL is present. | flat | @@ -445,7 +445,7 @@ Boundaries: - **amd-smi pre-flight.** The device pre-flight in [`amd_smi.rs`](../crates/rocm-dash-collectors/src/amd_smi.rs) is mandatory outside tests, because amd-smi can block in uninterruptible kernel sleep on a host with no readable `/dev/kfd`. Its one relaxation is the GPU-reachability verdict `apps/rocm` computes and passes in through the daemon's `RunnerOptions`. - **vLLM throughput.** [`vllm_prom.rs`](../crates/rocm-dash-collectors/src/vllm_prom.rs) leaves `gen_tps` empty: the daemon runner turns the generation-token counter into a rate. - **Engine label.** [`bench_load.rs`](../crates/rocm-dash-collectors/src/bench_load.rs) takes the `vllm` label from `EngineKind`, so the CSV `engine` column cannot drift from the registry's spelling. Because `engine` is a rollup key, a cell that mixes blank-engine and vLLM rows is reported through `EngineSplitNotice` for the CLI to print, not printed here. -- **Lemonade is off by default.** The daemon runs the Lemonade collector only when `RunnerOptions::enable_lemonade` is set; it defaults to false and the daemon `rocm dash` starts leaves it off. +- **Lemonade discovery follows the dashboard config.** The daemon runs the Lemonade collector only when `RunnerOptions::enable_lemonade` is set. `RunnerOptions::default()` leaves it off, for standalone use and tests; `rocm dash` sets it from `dashboard.daemon.enable_lemonade`, which defaults to true, so on Linux and WSL the daemon probes the configured Lemonade host and port (default `127.0.0.1:13305`) read-only unless the key is `false`. ### `crates/rocm-dash-daemon` — telemetry daemon @@ -457,8 +457,9 @@ The daemon library: [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs) driv | [`bench_ring.rs`](../crates/rocm-dash-daemon/src/bench_ring.rs) | `BenchRing`, the bounded benchmark-row history a late-joining client is sent when it subscribes. | flat | | [`demo.rs`](../crates/rocm-dash-daemon/src/demo.rs) | The synthetic-session generator behind `rocm dash --demo`: a deterministic recording of an 8-GPU node serving three vLLM containers, written as `PersistedEntry` lines. | flat | | [`persist.rs`](../crates/rocm-dash-daemon/src/persist.rs) | `SessionWriter`: appends every broadcast `Event` as a `PersistedEntry` line to one `session-*.ndjson` file per daemon session. | flat | -| [`registry.rs`](../crates/rocm-dash-daemon/src/registry.rs) | Reads the managed-service records `rocm serve` writes and turns the live ones into `DiscoveredService` scrape targets, telling a registry read in full, an empty or absent one included, from one that could not be read in full. It also keeps each live service's recorded engine and supervisor PIDs (with their start times) and derives the ancestry roots `ancestry.rs` matches GPU processes against, dropping the roots of a service it read and found stopped even on a partial pass. | flat | +| [`registry.rs`](../crates/rocm-dash-daemon/src/registry.rs) | Reads the managed-service records `rocm serve` writes and turns the live ones into `DiscoveredService` scrape targets, telling a registry read in full, an empty or absent one included, from one that could not be read in full. It also keeps each live service's recorded engine and supervisor PIDs (with their start times) and derives the ancestry roots `ancestry.rs` matches GPU processes against, dropping the roots of a service it read and found stopped even on a partial pass. It also turns the managed Lemonade records into read-only telemetry targets and decides which of them name the same endpoint (`endpoints_match`, `scrape_host`, `cached_collector_reusable`): `::` covers both loopback families, `::1` only IPv6, and `localhost`, an empty host, `127.0.0.1` and `0.0.0.0` only IPv4, and a cached collector is reused only for the address it dials. | flat | | [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs) | The owner task: drives the collectors on their tick cadences, reconciles discovered services, derives rates and efficiency, and broadcasts `Snapshot`s. | flat | +| [`runner/discovery.rs`](../crates/rocm-dash-daemon/src/runner/discovery.rs) | A private child of [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs), split out to keep it within the module-size limit: what the discovery passes remember between ticks (`TelemetryState`, `DiscoveryState` with the external Lemonade rows and the registry's Lemonade targets, `RegistryHealth`) and the `Reconcile` steps every discovery source shares. | domain | | [`server.rs`](../crates/rocm-dash-daemon/src/server.rs) | The listener, the per-client task loop and the broadcast from the runner to every client, including the history replay on subscribe. | flat | | [`snapshot_ring.rs`](../crates/rocm-dash-daemon/src/snapshot_ring.rs) | `SnapshotRing`, the bounded snapshot history a late-joining client is sent when it subscribes. | flat | | [`transport.rs`](../crates/rocm-dash-daemon/src/transport.rs) | NDJSON framing for the daemon's socket: `write_line` and `read_line`. | flat | @@ -552,7 +553,7 @@ The dashboard client: the event loop and reducer (`app/`), the chat agent (`agen | [`ui/quit_confirm.rs`](../crates/rocm-dash-tui/src/ui/quit_confirm.rs) | The quit-confirm prompt's content (`quit_confirm_request`): the title, the body listing what is still served, and the stop hint. | flat | | [`ui/runtime_manager.rs`](../crates/rocm-dash-tui/src/ui/runtime_manager.rs) | The Runtimes overlay: lists the registered runtimes and runs the `rocm runtimes` verbs, the mutating ones behind an approval. | flat | | [`ui/serve_wizard.rs`](../crates/rocm-dash-tui/src/ui/serve_wizard.rs) | The Serve form: builds a model-first `rocm serve` invocation and launches it as a job behind an approval. | flat | -| [`ui/services_manager.rs`](../crates/rocm-dash-tui/src/ui/services_manager.rs) | The Services overlay: lists the managed inference services with live throughput, and stops or restarts one behind an approval. | flat | +| [`ui/services_manager.rs`](../crates/rocm-dash-tui/src/ui/services_manager.rs) | The Services overlay: lists the managed inference services with their throughput (live, or a Lemonade server's last-request rate under a legend), and stops or restarts one behind an approval. | flat | | [`ui/sparkline.rs`](../crates/rocm-dash-tui/src/ui/sparkline.rs) | `BrailleSparkline`, a history graph that packs two samples into each character column at four dot rows per cell. | flat | | [`ui/spinner.rs`](../crates/rocm-dash-tui/src/ui/spinner.rs) | Pure helpers for the braille spinner frame (`spinner_frame`) and progress-percentage parsing (`parse_progress_pct`) the job console renders with. | flat | | [`ui/theme.rs`](../crates/rocm-dash-tui/src/ui/theme.rs) | The semantic `Theme` and its catalogue, and the shared status-to-color mapping (`StatusTone`, `tone_color`, `job_status_color`, `instance_status_color`). | flat | @@ -656,21 +657,23 @@ The Lemonade engine adapter (`rocm-engine-lemonade`): installs the embeddable Le | [`archive_cache.rs`](../engines/lemonade/src/archive_cache.rs) | The cached embeddable-archive layer: SHA-256 verification, verified publication into the cache, and the per-archive lock `ensure_cached_archive` returns as an `ArchiveCacheGuard` to hold while using the bytes. | domain | | [`archive_extract.rs`](../engines/lemonade/src/archive_extract.rs) | The hardened in-process tar.gz and zip extractor for the embeddable archive, and the containment-checked `copy_tree` into the runtime directory. | domain | | [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs) | Aligning the llama.cpp backend with the ROCm SDK (Tier 1, Tier 2, revert), including the `backend_versions.json` pin and setting ROCm backend directories aside around a forced reinstall. | domain | -| [`device_policy.rs`](../engines/lemonade/src/device_policy.rs) | Device policy and GPU selection: the GPU-required check, `--device`/`--gpu` parsing, the GPU ordinals to pin before a launch, and the device availability `detect` reports. | domain | -| [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) | Hugging Face checkpoint and GGUF resolution, and the direct serve path that runs a packaged `llama-server` on a GGUF (`serve_direct_llama_server`). | domain | +| [`device_policy.rs`](../engines/lemonade/src/device_policy.rs) | Device policy and GPU selection: the GPU-required check, `--device`/`--gpu` parsing, the GPU ordinals to pin before a launch, the device availability `detect` reports, and the notice that names a non-ROCm GPU backend (Vulkan) as the one serving. | domain | +| [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) | Hugging Face checkpoint and GGUF resolution, including a registered model name resolved through the installed Lemonade catalog, which route a name serves by, and the direct serve path that runs a packaged `llama-server` on a GGUF (`serve_direct_llama_server`). | domain | | [`footprint.rs`](../engines/lemonade/src/footprint.rs) | What a Lemonade install costs on disk: the pre-install notice and free-space check, and the read-only view of Lemonade's install trees and record. | domain | | [`install.rs`](../engines/lemonade/src/install.rs) | Embeddable-runtime download, extraction orchestration and manifest persistence, plus the `LemonadeRuntime` handle and `resolve_runtime` over the persisted manifest. | domain | | [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs) | llama.cpp backend discovery and selection: asks `lemonade backends` what the host supports, picks and installs the best GPU backend, and finds its `llama-server`. | domain | | [`process.rs`](../engines/lemonade/src/process.rs) | `lemond` spawn, environment and path setup, readiness and health polling, and the background relaunch of the `serve-http` command. | domain | +| [`router_serve.rs`](../engines/lemonade/src/router_serve.rs) | The router serve path: spawn `lemond`, select a GPU backend, load and verify the model by its router name, and on a failure after the spawn stop the server and settle the state. A load or verification failure may then be served directly; a probe that could not reach the endpoint is not one. | domain | | [`runtime_dir.rs`](../engines/lemonade/src/runtime_dir.rs) | `XDG_RUNTIME_DIR` tier 1, 2 and 3 hardening for the spawned child. | domain | | [`state.rs`](../engines/lemonade/src/state.rs) | Service state file I/O and port and endpoint helpers. | domain | +| [`test_http.rs`](../engines/lemonade/src/test_http.rs) | Test-only: a loopback HTTP fixture that answers each request with a chosen status and JSON body and records what it received, plus (Linux) fake `lemond` and `lemonade` executables that log their arguments. | flat | | [`test_support.rs`](../engines/lemonade/src/test_support.rs) | Test-only fixtures shared by the crate's module tests. | flat | Boundaries: - **GPU required, no CPU fallback.** [`device_policy.rs`](../engines/lemonade/src/device_policy.rs) refuses a CPU-only request rather than downgrading it, and [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs) prefers `rocm`, then `vulkan`, and never picks `cpu`. - **Layering.** - - Outside test code the modules form no import cycle: [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs) uses [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs), which uses [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs), which uses [`process.rs`](../engines/lemonade/src/process.rs), and none of them imports back. + - Outside test code the modules form no import cycle: [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs) uses [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs), which uses [`process.rs`](../engines/lemonade/src/process.rs); [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) uses both of those, and [`router_serve.rs`](../engines/lemonade/src/router_serve.rs) uses only `process.rs` for its types. `process.rs` imports none of them, and none imports back. - [`install.rs`](../engines/lemonade/src/install.rs) owns `LemonadeRuntime` and `resolve_runtime` so that [`state.rs`](../engines/lemonade/src/state.rs), [`process.rs`](../engines/lemonade/src/process.rs) and [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) reach the manifest without importing [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs). - **Shared test support.** The archive property-test support (`archive_props`) is included once at the crate root, so [`archive_extract.rs`](../engines/lemonade/src/archive_extract.rs) and [`install.rs`](../engines/lemonade/src/install.rs) both reach it as `crate::archive_props`. - **Exports for `apps/rocm`.** [`footprint.rs`](../engines/lemonade/src/footprint.rs)'s install-tree view is re-exported at the crate root for `rocm storage` and `rocm runtimes uninstall`. @@ -781,13 +784,16 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`src/binary_path.rs`](../tests/e2e-cucumber/src/binary_path.rs) | Pins a relative `ROCM_CLI_BINARY` path before a scenario runs the child from another directory, and leaves a bare name for `PATH` to resolve. | flat | | [`src/capability.rs`](../tests/e2e-cucumber/src/capability.rs) | Host capability probe: runs `rocm examine` and `rocm engines list` once, and re-derives the serve engine the CLI would pick and the AMD GPU count. | flat | | [`src/chat_request.rs`](../tests/e2e-cucumber/src/chat_request.rs) | The chat round trip against a served endpoint (model discovery, then one chat completion), run on its own task so its timeout measures the server. | flat | +| [`src/dash_screen.rs`](../tests/e2e-cucumber/src/dash_screen.rs) | Pure matching over captured dashboard screens and config files, so the logic the dashboard scenarios depend on has unit tests. | flat | | [`src/device_memory.rs`](../tests/e2e-cucumber/src/device_memory.rs) | Which device memory pool (VRAM, or GTT-backed memory on an APU) the drain-wait between serve scenarios judges, how it is sampled, and the free-memory floor. | flat | | [`src/expectation.rs`](../tests/e2e-cucumber/src/expectation.rs) | Resolves each scenario on this host to expected-pass, expected-fail or skip from its tags, the capability probe and the [`expectations.toml`](../tests/e2e-cucumber/expectations.toml) matrix. | flat | | [`src/fixture_binary.rs`](../tests/e2e-cucumber/src/fixture_binary.rs) | Locates a fake-subprocess `[[bin]]` of this crate next to the running test binary. | flat | | [`src/harness_diagnostics.rs`](../tests/e2e-cucumber/src/harness_diagnostics.rs) | Records a panic that escapes the cucumber run, which cucumber's silenced panic hook would hide, and the state of the standard streams, to stderr and a results-dir log. | flat | | [`src/http_server.rs`](../tests/e2e-cucumber/src/http_server.rs) | Shared `axum` test-server plumbing: bind an ephemeral loopback port, serve a router, shut down on drop. | flat | | [`src/installer_fixture.rs`](../tests/e2e-cucumber/src/installer_fixture.rs) | `with_pinned_release_keys`: a copy of the PowerShell installer with its two pinned-key assignments replaced, failing on any shape it does not expect. | flat | +| [`src/lemonade_backend.rs`](../tests/e2e-cucumber/src/lemonade_backend.rs) | Where an installed Lemonade runtime keeps a backend's `llama-server`: the directories searched for the selected backend, and a CPU or unknown backend rejected rather than found. | flat | | [`src/lemonade_diagnostics.rs`](../tests/e2e-cucumber/src/lemonade_diagnostics.rs) | Read-only backend evidence for a failed Lemonade serve, read from disk before the service stops and probed with `llama-server` after. | flat | +| [`src/lemonade_protocol.rs`](../tests/e2e-cucumber/src/lemonade_protocol.rs) | A loopback fixture that speaks the Lemonade health and stats protocol for the external-discovery scenarios, and records any request beyond reads. | flat | | [`src/loopback_http.rs`](../tests/e2e-cucumber/src/loopback_http.rs) | Loopback file server (`tower_http`'s `ServeDir`) that install, artifact and release-index scenarios point the real installer or CLI at. | flat | | [`src/mock_server.rs`](../tests/e2e-cucumber/src/mock_server.rs) | The mock OpenAI-compatible server (fixed, scripted, rejecting or metrics-serving) and the writer for planted managed-service records. | flat | | [`src/model_id.rs`](../tests/e2e-cucumber/src/model_id.rs) | `model_ids_match`: whether the model id a server reports names the requested model, across Lemonade file names and vLLM repo ids. | flat | diff --git a/docs/manual-testing.md b/docs/manual-testing.md index eb3f8de017..08f0719975 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -367,8 +367,9 @@ rocm serve qwen --engine lemonade --device gpu_required --foreground --port 1143 Expected result: -- The engine uses ROCm GPU execution. -- The run does not fall back to CPU or Vulkan. +- The engine uses its selected supported GPU backend, ROCm or Vulkan. +- The run does not fall back to CPU, and does not select a stale ROCm binary + when the installed runtime selects Vulkan. - If the GPU cannot be used, the command fails with a clear error. - The OpenAI-compatible endpoint answers a simple chat request. - `rocm services list` shows an `anthropic endpoint:` URL, and `POST /v1/messages` with `{"model":"x","max_tokens":16,"messages":[{"role":"user","content":"Hello"}]}` returns an Anthropic `message` that names the served model. The same request with `"stream":true` answers with `text/event-stream` (`message_start` through `message_stop`, text in `content_block_delta` events). A request with `tools` and `"tool_choice":{"type":"tool","name":""}` returns a `tool_use` content block naming that tool with its arguments and `"stop_reason":"tool_use"`; sending the call back as a `tool_result` block gets the model's answer. Combining `"stream":true` with `tools` is refused with an `invalid_request_error`. @@ -567,6 +568,107 @@ promise on the Windows GPU lane with whatever driver that machine has. Only the old-driver branch needs this manual check: it needs a machine whose display driver is dated before 27 July 2023. The logic is also covered by unit tests. +### Named models and dashboard discovery + +On Linux or WSL, repeat the serving and inference check with a non-default +registered model and with its explicit checkpoint: + +```bash +rocm serve qwen-smoke --engine lemonade --managed --port 11435 +# Stop the test-owned service before reusing its port. +rocm services stop --yes +rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0 --engine lemonade --managed --port 11435 +``` + +`rocm serve` returns to the shell once the server is ready. Background is the +default, and `--managed` is accepted as an explicit synonym. Only `--foreground` +or `--verbose` attach to the log stream. Use `rocm services` to find the service id. + +For each run, confirm that the endpoint answers inference for the requested +model and that `rocm dash` shows the managed model. A startup announcement alone +is not proof of a working endpoint. Stop only the services created by this check. +Simple single-GGUF catalog entries can use direct serving on Linux; registered +entries needing extra files or model-specific options stay with the router, as +does a registered checkpoint that is not a plain `owner/repo:variant`, such as +one whose variant is `*`. +Explicit checkpoints require the selected GPU backend and must not fall back to +another backend or CPU when it is missing. + +For the complex registered-model check, stop your previous test service and run: + +```bash +rocm serve Qwen3.5-0.8B-GGUF --engine lemonade --managed --port 11435 +``` + +This catalog entry needs both its GGUF checkpoint and projector file. Confirm +that inference identifies the requested model and the engine state records +`load_response.method` as `lemonade-cli`, not a packaged direct-server launch. +Stop that service with `rocm services stop --yes` and confirm its +endpoint closes. The automated equivalent is the nightly GPU scenario +`serve-lemonade-complex-registered-inference` in +[model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature). + +For a simple registered name on Linux, such as `Qwen3-0.6B-GGUF`, the engine +state records `load_response.method` as `lemonade-packaged-llama-server` and a +non-CPU `backend_requested`. The automated equivalent is +`serve-lemonade-simple-registered-direct-route` in the same feature file. + +For external discovery on Linux or WSL, use a separately running Lemonade server +with a loaded model. The embedded dashboard daemon probes `127.0.0.1:13305` by +default. For a non-default endpoint, set these fields in the CLI's `config.json` +(`~/.rocm/config.json`, or `config.json` in the directory named by +`ROCM_CLI_CONFIG_DIR`), preserving other configuration: + +```json +{ + "dashboard": { + "daemon": { + "enable_lemonade": true, + "lemonade_host": "127.0.0.1", + "lemonade_port": 13305 + } + } +} +``` + +Confirm the following through the dashboard and the external server: + +- Each loaded model appears, without creating a managed service record. +- With multiple loaded models, throughput is unavailable because endpoint-level + last-request statistics cannot be attributed safely. A managed direct + `llama-server` also shows unavailable throughput when it has no Lemonade stats API. +- A managed Lemonade server that answers its health route with 401 or 403 (API + key required) stays listed with its registry status, no throughput, and no + repeating health warning. External discovery still warns on such a response. +- A shown rate is the most recent completed request's rate and does not decay + while the server is idle. The Instances table shows the legend `Lemonade: last req + tok/s, not live` and the instance detail reads `last req: ` + instead of `gen: `; a managed Lemonade row in the Services overlay reads + `req ` instead of `gen ` under the same legend, and its quick preview + reads `last req `. A vLLM row shows neither. The row's tok/W shows `—`, + not an efficiency computed from that rate. With only an idle Lemonade server + loaded, the Home and Observe tok/s and tok/W heroes show no throughput instead + of repeating that rate. At 80 columns the legend, the instance detail's freshness + line and the empty Instances state are all still readable. +- The engine pane distinguishes managed installation from external availability, + even when GPU telemetry is unavailable. It reads the managed-runtime status once, + when the dashboard starts. After you install Lemonade from the Engines overlay, + restart the dashboard to see the new status. When the status cannot be read, the + pane says "unknown, run `rocm engines list`", and that command prints each + engine's detect result. +- Unloading a model removes it from the running-model list; reloading restores it. + A healthy but unloaded server does not claim a running model. +- In Observe, open Services with `s`. External models are not managed-service + targets. With only external models, the list reports no managed services; + pressing `s` or `r` stages no approval and starts no job. Close the list with + Escape and confirm the external endpoint is still reachable. +- Closing the dashboard also leaves the external server reachable. +- Setting `enable_lemonade` to `false` disables external discovery on the next + embedded-daemon launch; managed service records remain visible. + +Native Windows has no live dashboard telemetry. These dashboard checks apply to +Linux and WSL, not the native Windows CLI. + ## 4. Local Server Records After a managed or foreground serve attempt, inspect local server records: @@ -953,7 +1055,9 @@ Expected result: - From the dashboard (not the launcher), the prompt also offers `s`: pressing it closes the prompt and opens the Services overlay, pre-focused on the serving model, so you can stop it there through the overlay's own - approval+job flow. The launcher's copy of the prompt does not show this + approval+job flow. The offer appears once the dashboard lists the service as + managed; for a service started a moment earlier it is absent and the printed + stop command is the way to stop it. The launcher's copy of the prompt does not show this hint or respond to `s` — it has no live Services overlay to jump to — and neither does the dashboard on Windows, for the same reason the model's name is missing there. diff --git a/docs/testing.md b/docs/testing.md index 127d45c6bb..befbcc05ca 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -686,6 +686,193 @@ behaviour wanted, so un-ignoring it is the fix's acceptance test. A gap in user-observable CLI behaviour that an e2e scenario covers is tracked with an xfail row in `tests/e2e-cucumber/expectations.toml` instead. +## Lemonade serving and dashboard regressions + +The real GPU scenarios `serve-lemonade-inference` and +`serve-hf-checkpoint-inference` in +[model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature) +start Lemonade, request inference, and verify the model identity. The named-model +case uses `Qwen3-0.6B-GGUF`; the checkpoint case uses +`unsloth/Qwen3-0.6B-GGUF:Q4_0`. Both require a usable AMD GPU. Unit tests cover +registered-name catalog resolution, conservative routing of multi-file and +model-specific catalog entries, routing of a registered checkpoint that is not +a plain `owner/repo:variant` (for example, variant `*`) to the router, +quantization matching, and selection of the +runtime's GPU backend when competing backend directories exist. Engine tests also +verify persisted router identity through the health inference request; direct +servers retain the exact requested alias. + +`serve-lemonade-complex-registered-inference` covers the pinned catalog's +`Qwen3.5-0.8B-GGUF` entry, which also needs a projector file. It verifies router +state, real inference for that model, and stopping the scenario-owned managed +service on its isolated port. The `@requires-gpu` and +`@requires-engine:lemonade` tags require a usable AMD GPU on a Lemonade lane. +Its `@nightly` tag also requires `E2E_INCLUDE_NIGHTLY=1`, or the self-hosted +workflow's `include_nightly` dispatch input. A runtime channel named `nightly` +does not enable these scenarios by itself. + +`serve-lemonade-simple-registered-direct-route` is the Linux counterpart for a +simple registered entry (`Qwen3-0.6B-GGUF`: one checkpoint with a variant, no extra +files). It reads the engine state and asserts that `load_response.method` is +`lemonade-packaged-llama-server` and that the `llama-server` that served belongs to +the backend Lemonade selected (a stale directory of the other backend serving in its +place fails it), so a change of route is caught even when the endpoint still answers. It also runs +inference and stops the scenario-owned service. It needs a usable AMD GPU on a +Lemonade Linux lane (`@requires-gpu`, `@requires-os:linux`, +`@requires-engine:lemonade`). It has no flaky expectations row, so a route +regression fails the lane. Of the other direct-route serves, `serve-07` keeps its +flaky Linux row; `serve-08` has none, so it fails the lane too. + +The daemon's Lemonade contracts in +[telemetry_contract.rs](../crates/rocm-dash-daemon/tests/telemetry_contract.rs) +hold observation time during singleton-to-multiple-model transitions. The first +multi-model snapshot must clear both the rate and its observation metadata; +natural expiry cannot satisfy that assertion. Independent endpoints must overlap +while each endpoint checks health before fetching stats. Health failures include +endpoint context, and an unavailable external endpoint removes its rows without +a grace period. A never-seen endpoint remains quiet when nothing answers, +which means a refused connection or a timeout (Windows hangs on a closed local +port instead of refusing). A never-seen endpoint that answers with an error +status or a broken body warns. + +The dashboard scenarios in +[dash.feature](../tests/e2e-cucumber/features/dash.feature) include: + +- `dash-lemonade-external-protocol-discovery`: a Lemonade protocol fixture on an + isolated non-default port, with multiple loaded models and no managed records. + The real CLI, embedded daemon, and TUI must discover it, show unknown + throughput for each model, report installation truthfully, and exclude it from + Services lifecycle targets. Pressing stop and restart must leave no approval + prompt and start no job, and the models stay displayed. Twice, after those + keys and after dashboard quit, one step asserts that no managed record exists + and that the fixture saw no request other than a `GET` of its health or stats + route. The fixture is an in-process thread, so the dashboard cannot stop it; + the request log is what catches a load, unload, or stop sent to the external + server. +- `dash-lemonade-external-protocol-transitions`: loaded-model changes and endpoint + unavailability must update the dashboard, rather than retain false serving + claims. One remaining model shows its throughput, an empty or unreachable + server shows no models, and a reload restores both. +- `dash-lemonade-managed-inference-journey`: a GPU-gated real Lemonade serve and + inference followed by dashboard observation. On Linux Lemonade hosts it is + expected to be flaky and its serve is bounded by `serve_timeout_secs` (readiness polling; the CLI process is killed only 300 s after that) in + [expectations.toml](../tests/e2e-cucumber/expectations.toml). The model must + appear as a full row in Observe, and its instance detail must show it READY + on the port it was served on. This is real engine acceptance; + the protocol fixtures do not prove GPU inference or engine installation. + The flaky row matches every lane that can run this scenario, so the whole + journey is advisory: its dashboard, instance detail, and post-quit inference + checks cannot block a merge. Only the serve step is intermittent, but + expectations apply per scenario. It also does not gate the + "Lemonade — managed runtime installed" wording. + Unit tests gate that wording instead: + `managed_installed_verdict_follows_detect_result` in + [dash.rs](../apps/rocm/src/dash.rs), and the engine pane glyph and verdict + tests in [pane.rs](../crates/rocm-dash-tui/src/ui/tabs/pane.rs). No other + scenario shows the installed state through a real Detect without a serve. + `dash-lemonade-external-protocol-discovery` and + `dash-lemonade-external-protocol-transitions` show only the not-installed + state. + +`dash-07` also covers the positive half of the ownership rule. With a live +managed service, the Services overlay lists it with its port, and the stop key +stages an approval prompt that names it. The scenario declines the prompt, so +nothing is stopped. + +The isolated dashboard scenarios turn Lemonade discovery off unless a scenario +sets it, so a Lemonade server running on the default port of a self-hosted +runner cannot add rows or warnings. The two external protocol scenarios turn it +on with their own host and port. None of them exercises the unset-key default, +which a unit test in `apps/rocm` covers. + +`runtime-lemonade-backend-alignment-reported` tests the advice the direct +serving error gives for a missing `llama-server`, `rocm engines install lemonade +--reinstall`. It installs Lemonade, deletes the selected backend's `llama-server` +(asserting it was there and is gone), reinstalls, and requires the file to be +back. It needs a GPU and a real backend download, and it is tagged +`@requires-os:windows` and `@nightly`, so it runs only on the Windows nightly GPU +lane. The Linux direct route, where the direct serving error mostly fires, is not +exercised by it. Only the selected backend's binary is deleted; a general +damaged-runtime repair is not tested. + +`dash-lemonade-external-protocol-transitions` also requires the `last req tok/s` +legend next to the single remaining model's rate. The Instances table and detail +labels are covered by `table_labels_a_last_request_rate_and_only_then` in +[instances.rs](../crates/rocm-dash-tui/src/ui/tabs/instances.rs) and +`detail_names_a_last_request_rate` in +[instance_detail.rs](../crates/rocm-dash-tui/src/ui/instance_detail.rs). That such a rate +is not counted as live throughput is covered by unit tests: +`node_efficiency_leaves_out_a_last_request_rate` in +[efficiency.rs](../crates/rocm-dash-core/src/efficiency.rs), +`home_heroes_leave_out_a_last_request_rate` in +[home.rs](../crates/rocm-dash-tui/src/ui/tabs/home.rs), +`observe_heroes_leave_out_a_last_request_rate` in +[observe.rs](../crates/rocm-dash-tui/src/ui/tabs/observe.rs) and +`instance_tools_carry_the_last_request_flag` in +[snapshot.rs](../crates/rocm-dash-tui/src/agent/snapshot.rs). +The daemon's two IPv6 contract tests in +[telemetry_contract.rs](../crates/rocm-dash-daemon/tests/telemetry_contract.rs) fail +when the machine cannot bind an IPv6 loopback address, since they are the only cover +for the `::` to `::1` mapping; set `ROCM_TEST_NO_IPV6=1` on a machine that has none, +and they report that they did not run. + +A direct llama-server serve on a non-ROCm GPU backend logs that Lemonade is +serving on it only after the server is ready. `a_direct_serve_that_never_becomes_ready_does_not_log_that_it_is_serving` +and the `NotReady` case of `direct_server_step_failure_stops_the_child_and_marks_state_failed` +in [direct_llama.rs](../engines/lemonade/src/direct_llama.rs) assert the log and the failed +state together; both are Linux-only unit tests. + +A known external Lemonade endpoint that starts answering 401 or 403 on a +scrape-only tick is reported on that tick, not at the next discovery tick +(`lemonade_scrape_only_tick_warns_when_a_known_external_endpoint_requires_credentials` +in [telemetry_contract.rs](../crates/rocm-dash-daemon/tests/telemetry_contract.rs)). + +Resolving a split (sharded) GGUF to its first shard, and preferring a complete set +over an older partial download, have no Gherkin scenario. They need a real +multi-shard checkpoint (several GiB) or a staged Lemonade runtime that the harness +cannot fabricate. The unit tests in +[direct_llama.rs](../engines/lemonade/src/direct_llama.rs) cover them +(`parse_gguf_shard`, `first_shard_of_complete_set`, `is_incomplete_shard_set`). + +The Linux fallback from a failed Lemonade router load to direct serving has no +Gherkin scenario: forcing the router to reject a model needs a real `lemond` +failure that no safe, deterministic e2e hook produces. Unit tests in +[lib.rs](../engines/lemonade/src/lib.rs) and +[router_serve.rs](../engines/lemonade/src/router_serve.rs) cover it, asserting the +result and the state together: `router_load_failure_falls_back_to_the_selected_gpu_llama_server` +(llama-server arguments, final state, and the logged line naming the backend and +saying it is not ROCm), `only_a_router_load_failure_may_fall_back_and_only_on_linux`, +`fallback_never_serves_through_a_missing_or_cpu_backend`, and +`only_a_load_or_verification_failure_is_eligible_for_direct_fallback`. The +`serve-lemonade-complex-registered-inference` scenario remains the real-hardware +check of the router route. + +Unit tests in `rocm-dash-tui` cover the dashboard side of these scenarios. A +Services selection that outlives its row is clamped. A staged approval whose +target disappears is dropped, and the next key is handled as a normal key. The +engine pane shows the check or dot glyph for installed and not-installed +status. The `rocm dash --help` test asserts that Lemonade discovery is scoped to +Linux and WSL. + +Run the ordinary applicability-aware suite with `cargo xtask e2e`. For focused +protocol regression development on Linux or WSL: + +```bash +cargo xtask e2e -- -n 'External Lemonade protocol' +``` + +Cucumber name filtering bypasses the harness's custom applicability/expectation +filter. Use it only for scenarios applicable to the local host. Do not force +GPU-gated scenarios through a name filter on a GPU-less machine. Run the real +serving journeys through the normal GPU lane selection. + +A green local unit or protocol-fixture run does not clear the existing flaky +Linux serving expectations in +[expectations.toml](../tests/e2e-cucumber/expectations.toml). Removing one requires +verification of that same real serving scenario on suitable hardware. Report +GPU checks that could not run, and do not treat an installed version or a startup +message as proof that inference works. + ## Coverage floors Every workspace crate (except the `e2e-cucumber` harness) has a committed line-coverage @@ -2177,7 +2364,7 @@ so it cannot receive the value-typed `LEMONADE_API_KEY` that Lemonade's server reads. Rather than launch an unauthenticated public server, `rocm serve` **fails closed** on that exact combination (public bind + `--engine lemonade` + Windows) and directs the user to `--engine vllm` or a loopback host. vLLM (all platforms, -via `VLLM_API_KEY`), the packaged llama-server fallback (via `--api-key-file`), +via `VLLM_API_KEY`), direct llama-server serving (via `--api-key-file`), and Lemonade on non-Windows are all fully supported. Teaching the Windows detached-spawn primitive to carry a value-typed override (so Windows Lemonade can serve public with auth) is tracked as a separate follow-up. diff --git a/engines/aim/src/docker.rs b/engines/aim/src/docker.rs index bfb8f67631..94e95d6ee8 100644 --- a/engines/aim/src/docker.rs +++ b/engines/aim/src/docker.rs @@ -4,7 +4,7 @@ //! Thin wrapper around the `docker` CLI binary. No `bollard`/Docker API //! dependency: shelling out mirrors the pattern `engines/lemonade/src/process.rs` -//! already uses for its own CLI (`run_lemonade_pull`/`run_lemonade_backend_install`). +//! already uses for its own CLI (`run_lemonade_pull_with_parent`/`run_lemonade_backend_install`). use anyhow::{Context, Result, bail}; use std::collections::BTreeMap; diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index 3899d32f9c..dc07423f1d 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -22,11 +22,10 @@ use crate::llamacpp_backend::{ find_llama_server_binary_for_backend, install_best_llamacpp_backend, }; #[cfg(target_os = "linux")] -use crate::process::apply_lemonade_process_environment; +use crate::process::apply_lemonade_process_environment_with_parent; #[cfg(target_os = "windows")] use crate::process::lemonade_process_environment_vars; use crate::process::{LemonadeProcessEnvironment, lemonade_process_environment}; -#[cfg(target_os = "windows")] use crate::runtime_dir::ParentRuntimeEnvironment; /// Resource file inside the extracted Lemonade embeddable pinning which @@ -714,7 +713,13 @@ fn rocm_backend_resolves( .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - if apply_lemonade_process_environment(&mut command, process_env).is_err() { + if apply_lemonade_process_environment_with_parent( + &mut command, + process_env, + &ParentRuntimeEnvironment::current(), + ) + .is_err() + { return false; } let Ok(output) = command.output() else { @@ -874,7 +879,7 @@ const BACKEND_DIR_ASIDE_SUFFIX: &str = ".pre-alignment"; /// build into its own build-numbered subdirectory rather than replacing one in place, /// so a tier that installs a different `therock.version`/llama.cpp tag than a prior /// attempt (or the pinned default) can otherwise leave two builds side by side -- -/// after which [`crate::direct_llama::find_binary_in`]'s directory-order fallback, used by both alignment +/// after which [`crate::llamacpp_backend::find_binary_in`]'s directory-order fallback, used by both alignment /// verification and `rocm serve`, may resolve to whichever build a rejected tier /// installed instead of the one just verified. /// @@ -914,7 +919,7 @@ fn set_aside_rocm_llamacpp_backend_dirs( /// actually asided) before moving each `aside` back into place: a failed install can /// land its output under a *different* ROCm-family name than the one that was /// asided (Lemonade extracts each release into its own build-numbered directory -/// rather than replacing one in place), and [`crate::direct_llama::find_binary_in`]'s priority-ordered +/// rather than replacing one in place), and [`crate::llamacpp_backend::find_binary_in`]'s priority-ordered /// lookup would otherwise let that stray, unverified directory shadow the good /// build this function just restored. Best-effort: a failure here just risks the /// same stale-directory ambiguity the aside exists to prevent, not the reinstall diff --git a/engines/lemonade/src/device_policy.rs b/engines/lemonade/src/device_policy.rs index a56aa7aa84..00ca729d61 100644 --- a/engines/lemonade/src/device_policy.rs +++ b/engines/lemonade/src/device_policy.rs @@ -5,6 +5,7 @@ use anyhow::{Result, bail}; use rocm_engine_protocol::{DevicePolicy, EngineDeviceAvailability, GpuSelection}; use serde_json::Value; +use std::path::Path; pub(crate) fn normalize_device_policy(policy: Option) -> Result { match policy.unwrap_or(DevicePolicy::GpuRequired) { @@ -143,6 +144,38 @@ pub(crate) fn reported_device(state: Option<&Value>, backend: &str) -> String { } } +/// A one-line notice naming the backend actually serving when it is a GPU +/// backend other than ROCm (Vulkan), so it is never read as a ROCm serve. `None` +/// for ROCm backends. The text is derived from the same backend name the engine +/// records in its state (`backend_requested`), so it cannot disagree with it. +pub(crate) fn non_rocm_backend_notice(backend: &str) -> Option { + let backend = backend.trim(); + (!backend.to_ascii_lowercase().starts_with("rocm")).then(|| { + format!( + "Lemonade is serving on the {backend} GPU backend, not ROCm; no CPU fallback is used." + ) + }) +} + +/// Print the non-ROCm backend notice and append it to the serve log, because a +/// managed serve runs with its stdout and stderr discarded and the log is the only +/// place the user can read it. Returns the notice, or `None` for a ROCm backend. +pub(crate) fn announce_non_rocm_backend(backend: &str, log_path: Option<&Path>) -> Option { + use std::io::Write as _; + let notice = non_rocm_backend_notice(backend)?; + eprintln!("{notice}"); + if let Some(log_path) = log_path + && let Some(mut log) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .ok() + { + let _ = writeln!(log, "{notice}"); + } + Some(notice) +} + /// The first pinned GPU ordinal recorded in service state, if any. fn first_gpu_index_from_state(state: &Value) -> Option { state diff --git a/engines/lemonade/src/direct_llama.rs b/engines/lemonade/src/direct_llama.rs index bd73f31290..ed975f763e 100644 --- a/engines/lemonade/src/direct_llama.rs +++ b/engines/lemonade/src/direct_llama.rs @@ -2,224 +2,380 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result, anyhow, bail}; +use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, format_http_base_url, http_get_text_with_auth, runtime_home_dir}; use serde_json::{Value, json}; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; -use std::process::{Command as ProcessCommand, Stdio}; +use std::process::{Child, Command as ProcessCommand, ExitStatus, Stdio}; use std::time::Duration; -use crate::install::{LemonadeInstallManifest, LemonadeRuntime, platform_binary_name}; +use crate::install::{LemonadeInstallManifest, LemonadeRuntime}; +use crate::llamacpp_backend::{backend_server_dirs, find_llama_server_binary_for_backend}; use crate::process::{ - LemonadeProcessEnvironment, apply_lemonade_process_environment, hide_child_console_window, - model_names_match, query_chat_smoke_endpoint, run_lemonade_model_load, run_lemonade_pull, - spawn_lemond, wait_for_lemonade_cli_status, + LemonadeProcessEnvironment, LogMode, apply_environment_vars, hide_child_console_window, + lemonade_process_environment_vars, query_chat_smoke_endpoint, run_lemonade_pull_with_parent, + spawn_lemond_with_parent, wait_for_lemonade_cli_status_with_parent, }; +use crate::runtime_dir::ParentRuntimeEnvironment; use crate::state::{ free_local_port, mark_json_status, merge_json_state, resolve_lemonade_model_ref, terminate_pid, write_running_state, }; -use crate::{ - DEFAULT_HOST, DEFAULT_MODEL, DEFAULT_MODEL_GGUF, DEFAULT_MODEL_REPO_DIR, ROCM_BACKEND_NAME, - ServeHttpRequest, -}; +use crate::{DEFAULT_HOST, ROCM_BACKEND_NAME, ServeHttpRequest}; + +/// Direct serving is only available for a local GGUF or an exact checkpoint. +/// Unsupported registered recipes stay on the Lemonade router path. +#[derive(Debug)] +pub(crate) enum DirectLlamaModel { + Local(PathBuf), + Checkpoint(HfCheckpoint), +} + +#[derive(Debug)] +pub(crate) enum ServeRoute { + Direct { + model: DirectLlamaModel, + server: PathBuf, + }, + Router { + model_ref: String, + }, +} -/// Serve a canonical Hugging Face checkpoint under its exact name. Requires an explicit -/// `:variant`; downloads the GGUF, then runs whichever packaged GPU llama-server backend -/// is installed (e.g. `vulkan` on WSL2, `rocm-stable` on native ROCm) directly on the -/// file with `--alias`. This bypasses Lemonade's model registry, whose naming rules -/// cannot preserve an `owner/repo:variant` name. -pub(crate) fn serve_hf_checkpoint( +/// Choose the serving path before downloading or launching a server. +pub(crate) fn serving_route( + manifest: &LemonadeInstallManifest, request: &ServeHttpRequest, - runtime: &LemonadeRuntime, - process_env: &LemonadeProcessEnvironment, - log_path: Option<&Path>, - checkpoint: &HfCheckpoint, -) -> Result<()> { - // A variant is required: a bare `owner/repo` would need Lemonade's interactive - // variant menu (which cannot be answered from this non-interactive path) and, if a - // GGUF happened to be cached, risks silently serving the wrong quantization. - if let Some(message) = hf_checkpoint_variant_required(&request.model_ref) { - bail!("{message}"); + is_linux: bool, +) -> Result { + crate::device_policy::require_gpu_required(&request.device_policy)?; + let explicit_checkpoint = parse_hf_checkpoint(&request.model_ref).is_some(); + // Windows serves a registered name through the router on purpose: direct + // serving passes `-m`, `--n-gpu-layers`, `--alias` and the recipe flags, and does + // not apply the router's per-model launch defaults, so only an explicit + // checkpoint is served directly there. + if (is_linux || explicit_checkpoint) + && let Some(model) = resolve_direct_llama_model(manifest, &request.model_ref)? + && let Some(server) = direct_serve_binary(manifest) + { + return Ok(ServeRoute::Direct { model, server }); } - let Some(server) = find_llama_server_binary(&runtime.manifest) else { - bail!( - "no GPU llama-server backend is installed under {}; run `rocm engines install lemonade`", - runtime - .manifest - .runtime_dir - .join("bin") - .join("llamacpp") - .display() - ); - }; - ensure_hf_checkpoint_downloaded(request, runtime, process_env, log_path)?; - // Never silently pick between quantizations: if the variant matches more than one - // cached GGUF, ask the user to disambiguate instead of choosing one arbitrarily. - let paths = AppPaths::discover()?; - let matches = hf_checkpoint_gguf_matches(&paths, checkpoint); - if matches.len() > 1 { - let names = matches - .iter() - .filter_map(|path| path.file_name().and_then(|name| name.to_str())) - .collect::>() - .join(", "); + if explicit_checkpoint { + if manifest.backend_recipe != crate::LLAMACPP_RECIPE { + bail!( + "selected Lemonade recipe `{}` does not support direct HF checkpoint serving; a GPU llama.cpp recipe is required", + manifest.backend_recipe + ); + } + if backend_server_dirs(&manifest.backend_name).is_none() { + bail!( + "selected Lemonade backend `{}` does not support GPU direct serving; no CPU fallback is used", + manifest.backend_name + ); + } + // The backend is a supported GPU one, so only its binary is absent. A + // reinstall re-extracts the runtime and reinstalls the best backend, which + // restores the file whatever state Lemonade recorded for it. bail!( - "variant `{}` matches multiple files ({names}); specify an exact quantization or filename, e.g. `{}:{}`", - checkpoint.variant.as_deref().unwrap_or_default(), - request.model_ref, - matches - .first() - .and_then(|path| path.file_name()) - .and_then(|name| name.to_str()) - .unwrap_or(".gguf") + "no selected GPU llama-server backend `{}` is installed under {} for direct serving of `{}`; run `rocm engines install lemonade --reinstall` to restore it", + manifest.backend_name, + manifest.runtime_dir.display(), + request.model_ref ); } - serve_direct_llama_server( - request, - runtime, - process_env, - &server, - log_path, - &anyhow!( - "serving canonical Hugging Face checkpoint `{}` directly", - request.model_ref - ), - ) + Ok(ServeRoute::Router { + model_ref: resolve_lemonade_model_ref(&request.model_ref), + }) +} + +/// The direct route still open after the router failed to load or verify a +/// model: Linux only, a model the registry resolves to a safe checkpoint, and the +/// llama-server of the backend Lemonade selected (read from the manifest the +/// router serve updated). A CPU or unknown backend never yields a server, so this +/// cannot become a CPU fallback. +pub(crate) fn direct_fallback_route( + manifest: &LemonadeInstallManifest, + request: &ServeHttpRequest, + is_linux: bool, +) -> Option<(DirectLlamaModel, PathBuf)> { + if !is_linux { + return None; + } + let model = resolve_direct_llama_model(manifest, &request.model_ref) + .ok() + .flatten()?; + let server = direct_serve_binary(manifest)?; + Some((model, server)) } -/// Ensure a canonical Hugging Face checkpoint's GGUF is in the HF hub cache, pulling -/// it through a short-lived `lemond` if needed. The bare `pull owner/repo:variant` -/// only downloads (and registers a derived name we ignore); the file is what we serve. -fn ensure_hf_checkpoint_downloaded( +/// Serve directly after the router failed to load or verify the model, or return +/// the router's error unchanged when no direct route is open. The line it prints +/// and logs names the backend taken from the server path it launches. +#[allow(clippy::too_many_arguments)] +pub(crate) fn serve_direct_after_router_failure( + paths: &AppPaths, request: &ServeHttpRequest, runtime: &LemonadeRuntime, process_env: &LemonadeProcessEnvironment, log_path: Option<&Path>, + parent: &ParentRuntimeEnvironment, + is_linux: bool, + router_error: anyhow::Error, ) -> Result<()> { - let paths = AppPaths::discover()?; - if direct_llama_model_path(&paths, &runtime.manifest.runtime_dir, &request.model_ref).is_some() - { - return Ok(()); - } - let download_port = free_local_port()?; - let mut child = spawn_lemond( - &runtime.manifest, - DEFAULT_HOST, - download_port, - log_path, + let Some((model, server)) = direct_fallback_route(&runtime.manifest, request, is_linux) else { + // No direct route is open, so the router's failure is final. The router + // left its state for this function to settle. + let _ = crate::state::mark_json_status(&request.state_path, "failed"); + return Err(router_error); + }; + let backend = llama_server_backend_label(&server); + let reason = format!( + "Lemonade router could not serve {} ({router_error:#}); serving it directly with the selected {backend} GPU llama-server", + request.model_ref + ); + eprintln!("{reason}"); + // The router left the state at `running`, with a server that is already + // stopped. A reader that sees it while the download or the spawn is still + // running would report a server that is gone, and one that sees `failed` + // would record the service as failed for good (`refresh_from_engine_state` + // ignores later states once a record is `failed`), so the state says + // `starting` for the whole fallback and goes to `failed` if the fallback + // does not produce a running server. The router's server pid goes too: + // `identity_from_state` prefers it over the launcher's, so leaving the dead + // router's pid in place would point a `stop` during the fallback at a + // process that is gone (or, once reused, at an unrelated one) instead of the + // launcher that is about to start the direct server. + crate::state::merge_json_state( + &request.state_path, + &serde_json::json!({ + "engine": crate::ENGINE_NAME, + "status": "starting", + "server_pid": null, + "server_start_ticks": null, + }), + ) + .with_context(|| { + format!( + "failed to record that the direct serving fallback is starting (the router had failed: {router_error:#})" + ) + })?; + let outcome = ensure_model_in( + paths, + request, + runtime, process_env, - )?; - let result = (|| -> Result<()> { - wait_for_lemonade_cli_status( - &runtime.manifest, - DEFAULT_HOST, - download_port, - Duration::from_secs(30), - log_path, + log_path, + &model, + parent, + ) + .with_context(|| format!("{router_error:#}; the direct serving fallback also failed")) + .and_then(|model_path| { + serve_direct_llama_server( + request, + runtime, process_env, - )?; - run_lemonade_pull( - &runtime.manifest, - DEFAULT_HOST, - download_port, - &request.model_ref, + &server, + &model_path, log_path, - process_env, - )?; - if direct_llama_model_path(&paths, &runtime.manifest.runtime_dir, &request.model_ref) - .is_some() - { - Ok(()) - } else { - bail!( - "Lemonade did not download `{}` (searched {})", - request.model_ref, - describe_hf_cache_roots(&paths) - ) + &reason, + parent, + ) + }); + if outcome.is_err() { + let _ = crate::state::mark_json_status(&request.state_path, "failed"); + } + outcome +} + +pub(crate) fn resolve_direct_llama_model( + manifest: &LemonadeInstallManifest, + model_ref: &str, +) -> Result> { + let path = PathBuf::from(model_ref); + if path.is_file() + && path + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf")) + { + return Ok(Some(DirectLlamaModel::Local(path))); + } + Ok(resolve_direct_llama_checkpoint(manifest, model_ref)?.map(DirectLlamaModel::Checkpoint)) +} + +/// Resolve the cached file without choosing an arbitrary quantization. The same +/// resolution is used before downloading and immediately before server launch. +pub(crate) fn direct_llama_model_path( + paths: &AppPaths, + model: &DirectLlamaModel, +) -> Result> { + match model { + DirectLlamaModel::Local(path) => Ok(path.is_file().then(|| path.clone())), + DirectLlamaModel::Checkpoint(checkpoint) => { + let matches = hf_checkpoint_gguf_matches(paths, checkpoint); + // A split model is one model: serve it from its first shard, which + // llama-server uses to load the remaining shards beside it. + if let Some(first) = first_shard_of_complete_set(&matches) { + return Ok(Some(first)); + } + // Some but not all shards of one model (an interrupted download): treat + // the model as absent so it is downloaded again, never served partial. + if is_incomplete_shard_set(&matches) { + return Ok(None); + } + if matches.len() > 1 { + let names = matches + .iter() + .filter_map(|path| path.file_name()) + .map(|name| name.to_string_lossy()) + .collect::>() + .join(", "); + let example = matches + .first() + .and_then(|path| path.file_name()) + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + bail!( + "variant `{}` matches multiple files ({names}); specify an exact quantization or filename, e.g. `{}/{}:{example}`", + checkpoint.variant.as_deref().unwrap_or_default(), + checkpoint.owner, + checkpoint.repo + ); + } + Ok(matches.into_iter().next()) } - })(); - let _ = terminate_pid(child.id(), true); - let _ = child.wait(); - result + } } -pub(crate) fn ensure_direct_llama_model_available( +/// Return the GGUF for `model`, downloading it first when absent. The cache +/// location and parent runtime environment are supplied by the caller. The hub +/// lookup also reads HF cache environment variables and the user's home directory; +/// tests use unique checkpoint names to avoid matching unrelated cached models. +pub(crate) fn ensure_model_in( + paths: &AppPaths, request: &ServeHttpRequest, runtime: &LemonadeRuntime, process_env: &LemonadeProcessEnvironment, log_path: Option<&Path>, -) -> Result<()> { - let paths = AppPaths::discover()?; - let runtime_dir = runtime.manifest.runtime_dir.as_path(); - if direct_llama_model_path(&paths, runtime_dir, &request.model_ref).is_some() { - return Ok(()); - } - - let download_port = free_local_port()?; - let mut child = spawn_lemond( - &runtime.manifest, - DEFAULT_HOST, - download_port, - log_path, - process_env, - )?; - let result = (|| -> Result<()> { - wait_for_lemonade_cli_status( + model: &DirectLlamaModel, + parent: &ParentRuntimeEnvironment, +) -> Result { + ensure_model_with_download(paths, &request.model_ref, model, |checkpoint_ref| { + let download_port = free_local_port()?; + let mut child = spawn_lemond_with_parent( &runtime.manifest, DEFAULT_HOST, download_port, - Duration::from_secs(30), log_path, process_env, + parent, + // The log may already hold why a router failed and this fallback is running. + LogMode::Append, )?; - let _ = run_lemonade_model_load( - &runtime.manifest, - DEFAULT_HOST, - download_port, - &request.model_ref, - ROCM_BACKEND_NAME, - None, - log_path, - process_env, - ); - if direct_llama_model_path(&paths, runtime_dir, &request.model_ref).is_some() { - Ok(()) - } else { - bail!( - "{}", - unresolved_model_message(&hf_cache_roots(&paths), runtime_dir, &request.model_ref) + let result = (|| -> Result<()> { + wait_for_lemonade_cli_status_with_parent( + &runtime.manifest, + DEFAULT_HOST, + download_port, + Duration::from_secs(30), + log_path, + process_env, + parent, + )?; + // Pull downloads without a model load, so this path neither guesses a + // backend nor masks load errors. + run_lemonade_pull_with_parent( + &runtime.manifest, + DEFAULT_HOST, + download_port, + checkpoint_ref, + log_path, + process_env, + parent, ) - } - })(); - let _ = terminate_pid(child.id(), true); - let _ = child.wait(); - result + })(); + let _ = terminate_pid(child.id(), true); + let _ = child.wait(); + result + }) +} + +/// Return the cached GGUF, downloading it first when absent. `download` receives +/// the validated checkpoint reference (`owner/repo:variant`), never the alias the +/// user typed, and its error is returned as is. +fn ensure_model_with_download( + paths: &AppPaths, + requested: &str, + model: &DirectLlamaModel, + download: impl FnOnce(&str) -> Result<()>, +) -> Result { + if let Some(path) = direct_llama_model_path(paths, model)? { + return Ok(path); + } + let DirectLlamaModel::Checkpoint(checkpoint) = model else { + bail!("local GGUF `{requested}` is missing"); + }; + download(&checkpoint.model_ref())?; + direct_llama_model_path(paths, model)?.with_context(|| { + format!( + "Lemonade did not download `{requested}` for direct serving (searched {})", + describe_hf_cache_roots(paths) + ) + }) } /// Run one of Lemonade's packaged `llama-server` binaries (`server`) directly on a /// GGUF file, exposing the model under `request.model_ref` via `--alias`. This is how /// a canonical Hugging Face name is served under exactly that name — Lemonade's router /// renames registered models, but an llama-server alias is a free-form string. +#[allow(clippy::too_many_arguments)] pub(crate) fn serve_direct_llama_server( request: &ServeHttpRequest, runtime: &LemonadeRuntime, process_env: &LemonadeProcessEnvironment, server: &Path, + model_path: &Path, + log_path: Option<&Path>, + reason: &str, + parent: &ParentRuntimeEnvironment, +) -> Result<()> { + serve_direct_llama_server_within( + request, + runtime, + process_env, + server, + model_path, + log_path, + reason, + parent, + DIRECT_SERVER_READY_TIMEOUT, + ) +} + +/// How long a direct llama-server may take to answer `/v1/models` and a chat smoke +/// test before the serve is treated as failed. +const DIRECT_SERVER_READY_TIMEOUT: Duration = Duration::from_mins(2); + +#[allow(clippy::too_many_arguments)] +fn serve_direct_llama_server_within( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + process_env: &LemonadeProcessEnvironment, + server: &Path, + model_path: &Path, log_path: Option<&Path>, - reason: &anyhow::Error, + reason: &str, + parent: &ParentRuntimeEnvironment, + ready_timeout: Duration, ) -> Result<()> { - let paths = AppPaths::discover()?; - let model_path = - direct_llama_model_path(&paths, &runtime.manifest.runtime_dir, &request.model_ref) - .with_context(|| { - format!( - "Lemonade downloaded model `{}` was not found for direct serving", - request.model_ref - ) - })?; + if !model_path.is_file() { + bail!( + "GGUF for `{}` is missing at {}", + request.model_ref, + model_path.display() + ); + } if !server.is_file() { bail!("Lemonade llama-server is missing at {}", server.display()); } @@ -238,7 +394,7 @@ pub(crate) fn serve_direct_llama_server( .with_context(|| format!("failed to open {}", log_path.display()))?; writeln!( log, - "\nLaunching Lemonade packaged {backend} llama-server directly: {reason:#}" + "\nLaunching Lemonade packaged {backend} llama-server directly: {reason}" ) .ok(); } @@ -249,22 +405,7 @@ pub(crate) fn serve_direct_llama_server( push_existing_path(&mut direct_env.library_entries, server_dir.to_path_buf()); } - let mut command = ProcessCommand::new(server); - command - .arg("-m") - .arg(&model_path) - .arg("--host") - .arg(&request.host) - .arg("--port") - .arg(request.port.to_string()) - .arg("--n-gpu-layers") - .arg("999") - .arg("--alias") - .arg(&request.model_ref) - .stdin(Stdio::null()); - if let Some(engine_recipe) = request.engine_recipe.as_ref() { - command.args(&engine_recipe.required_flags); - } + let mut command = direct_llama_server_command(request, server, model_path); // Packaged llama-server is raw llama.cpp — it does not read `LEMONADE_API_KEY`. // When `rocm serve` protects a public endpoint, hand llama-server the *existing* // CLI-managed 0600 key file via `--api-key-file` (a path, not the value, so the @@ -285,62 +426,137 @@ pub(crate) fn serve_direct_llama_server( } else { command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); } - apply_lemonade_process_environment(&mut command, &direct_env)?; + let vars = lemonade_process_environment_vars(&direct_env, parent)?; + apply_environment_vars(&mut command, &vars); hide_child_console_window(&mut command); let mut child = command .spawn() .with_context(|| format!("failed to start {}", server.display()))?; - write_running_state( + let mut steps = RealDirectServerSteps { request, runtime, - std::process::id(), - Some(child.id()), - "running", - )?; - // If the server never becomes ready (or fails the smoke test), don't leak the child - // or leave the state stuck at `running`: kill it and mark the service failed. - let readiness = (|| -> Result<()> { + backend: &backend, + model_path, + log_path, + ready_timeout, + }; + supervise_direct_server(&mut child, &request.state_path, &mut steps) +} + +/// The fallible steps that follow spawning a direct llama-server, so the failure +/// handling in [`supervise_direct_server`] is exercised without a live server. +trait DirectServerSteps { + fn record_running(&mut self, server_pid: u32) -> Result<()>; + fn wait_ready(&mut self) -> Result<()>; + fn record_ready(&mut self, server_pid: u32) -> Result<()>; + /// Announce what is now true. Called only once the server answered, passed its + /// chat smoke test and the `ready` state is written: an earlier announcement + /// would claim a serve that a failed readiness check then contradicts. + fn announce_serving(&mut self); + fn wait_exit(&mut self, child: &mut Child) -> std::io::Result { + child.wait() + } +} + +struct RealDirectServerSteps<'a> { + request: &'a ServeHttpRequest, + runtime: &'a LemonadeRuntime, + backend: &'a str, + model_path: &'a Path, + log_path: Option<&'a Path>, + ready_timeout: Duration, +} + +impl DirectServerSteps for RealDirectServerSteps<'_> { + fn record_running(&mut self, server_pid: u32) -> Result<()> { + write_running_state( + self.request, + self.runtime, + std::process::id(), + Some(server_pid), + "running", + ) + } + + fn wait_ready(&mut self) -> Result<()> { + let request = self.request; wait_for_openai_models_ready( &request.host, request.port, &request.model_ref, - Duration::from_mins(2), + self.ready_timeout, )?; if !query_chat_smoke_endpoint(&request.host, request.port, &request.model_ref)? { bail!("Lemonade packaged llama-server did not pass a chat-completion smoke test"); } Ok(()) - })(); - if let Err(error) = readiness { - let _ = terminate_pid(child.id(), true); + } + + fn record_ready(&mut self, server_pid: u32) -> Result<()> { + let request = self.request; + merge_json_state( + &request.state_path, + &json!({ + "status": "ready", + "server_pid": server_pid, + // Identity token for the server PID, captured while the child is alive. + "server_start_ticks": rocm_core::process_start_ticks(server_pid), + "backend_state": "ready", + "backend_requested": self.backend, + "backend_mode": "lemonade-packaged-llama-server", + "load_response": { + "status": "loaded", + "method": "lemonade-packaged-llama-server", + "model_name": request.model_ref, + "model_path": self.model_path, + "llamacpp_backend": self.backend + }, + }), + ) + } + + fn announce_serving(&mut self) { + crate::device_policy::announce_non_rocm_backend(self.backend, self.log_path); + } +} + +/// Own a just-spawned direct llama-server. Every step after the spawn, state +/// writes and the final wait included, runs under one guard: any failure stops and +/// reaps the child (`Child` does not kill on drop) and marks the state `failed`, +/// so the server can never keep running with no state that stop and health can +/// reach. +fn supervise_direct_server( + child: &mut Child, + state_path: &Path, + steps: &mut S, +) -> Result<()> { + let server_pid = child.id(); + let stop_and_fail = |child: &mut Child| { + let _ = terminate_pid(server_pid, true); let _ = child.wait(); - let _ = mark_json_status(&request.state_path, "failed"); + let _ = mark_json_status(state_path, "failed"); + }; + let started = (|| -> Result<()> { + steps.record_running(server_pid)?; + steps.wait_ready()?; + steps.record_ready(server_pid)?; + steps.announce_serving(); + Ok(()) + })(); + if let Err(error) = started { + stop_and_fail(child); return Err(error); } - merge_json_state( - &request.state_path, - &json!({ - "status": "ready", - "server_pid": child.id(), - // Identity token for the server PID, captured while the child is alive. - "server_start_ticks": rocm_core::process_start_ticks(child.id()), - "backend_state": "ready", - "backend_requested": backend, - "backend_mode": "lemonade-packaged-llama-server", - "load_response": { - "status": "loaded", - "method": "lemonade-packaged-llama-server", - "model_name": request.model_ref, - "model_path": model_path, - "llamacpp_backend": backend - }, - }), - )?; - let status = child - .wait() - .context("failed waiting for Lemonade packaged llama-server")?; + let status = match steps.wait_exit(child) { + Ok(status) => status, + Err(error) => { + stop_and_fail(child); + return Err(anyhow::Error::new(error) + .context("failed waiting for Lemonade packaged llama-server")); + } + }; mark_json_status( - &request.state_path, + state_path, if status.success() { "stopped" } else { @@ -354,6 +570,30 @@ pub(crate) fn serve_direct_llama_server( } } +pub(crate) fn direct_llama_server_command( + request: &ServeHttpRequest, + server: &Path, + model_path: &Path, +) -> ProcessCommand { + let mut command = ProcessCommand::new(server); + command + .arg("-m") + .arg(model_path) + .arg("--host") + .arg(&request.host) + .arg("--port") + .arg(request.port.to_string()) + .arg("--n-gpu-layers") + .arg("999") + .arg("--alias") + .arg(&request.model_ref) + .stdin(Stdio::null()); + if let Some(recipe) = request.engine_recipe.as_ref() { + command.args(&recipe.required_flags); + } + command +} + fn wait_for_openai_models_ready( host: &str, port: u16, @@ -401,7 +641,7 @@ fn models_payload_has_loaded_model(value: &Value, model_ref: &str) -> bool { ["id", "model", "name"] .into_iter() .filter_map(|field| model.get(field).and_then(Value::as_str)) - .any(|loaded| model_names_match(loaded, model_ref)) + .any(|loaded| loaded == model_ref) }) }) } @@ -423,6 +663,17 @@ pub(crate) struct HfCheckpoint { variant: Option, } +impl HfCheckpoint { + fn model_ref(&self) -> String { + format!( + "{}/{}:{}", + self.owner, + self.repo, + self.variant.as_deref().unwrap_or_default() + ) + } +} + /// Parse a model reference as a canonical Hugging Face checkpoint. Returns `None` for /// anything not of the form `owner/repo[:variant]` — built-in Lemonade aliases and /// bare registered names all fall through unchanged. @@ -463,7 +714,7 @@ pub(crate) fn parse_hf_checkpoint(model_ref: &str) -> Option { /// /// The refusal lives here, beside the parser, because two callers at very different /// depths need the same answer: the pre-launch `ResolveModel` the CLI makes before it -/// starts anything, and `serve_hf_checkpoint` inside the background engine process. +/// starts anything, and the direct serve inside the background engine process. /// Only one of the two is visible to the user on a given platform, so wording written /// out twice would drift and leave whichever one the user actually sees as the stale /// copy. @@ -502,42 +753,14 @@ fn is_safe_hf_component(component: &str) -> bool { .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')) } -/// Packaged llama.cpp backends whose `llama-server` we can drive directly, best first. -/// GPU backends only — `cpu` is intentionally excluded so the direct-serve path never -/// silently serves on CPU under a GPU-required policy (AGENTS.md §6). An explicit -/// allowlist rather than a directory scan, so the path is built only from constants. +/// Directory names under `bin/llamacpp/` that [`llama_server_backend_label`] +/// recognizes when it names the backend of a packaged `llama-server`. GPU backends +/// only; `cpu` is intentionally absent so a CPU build is never labelled as, or +/// mistaken for, a direct-serve backend. This is only a label lookup: which +/// directories are searched for a selected backend is decided by +/// [`crate::llamacpp_backend::backend_server_dirs`]. const DIRECT_LLAMA_SERVER_BACKENDS: [&str; 4] = ["rocm-stable", "rocm-nightly", "rocm", "vulkan"]; -/// Find an installed Lemonade `llama-server` binary under `bin/llamacpp//`, -/// checking known backends in priority order. This lets direct serving work wherever -/// Lemonade installed a backend (e.g. `vulkan` on WSL2), not just the ROCm build. -/// -/// Lemonade's backend installer extracts some llama.cpp releases straight into -/// `/`, but others land one level deeper under a build-numbered directory -/// (e.g. `rocm-stable/llama-b9752/llama-server`), so both layouts are checked. -pub(crate) fn find_llama_server_binary(manifest: &LemonadeInstallManifest) -> Option { - let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); - let binary = platform_binary_name("llama-server"); - DIRECT_LLAMA_SERVER_BACKENDS - .into_iter() - .find_map(|backend| find_binary_in(&llamacpp_dir.join(backend), &binary)) -} - -/// Look for `binary` directly in `dir`, then one level down in each subdirectory. -pub(crate) fn find_binary_in(dir: &Path, binary: &str) -> Option { - let direct = dir.join(binary); - if direct.is_file() { - return Some(direct); - } - fs::read_dir(dir) - .ok()? - .filter_map(Result::ok) - .map(|entry| entry.path()) - .filter(|path| path.is_dir()) - .map(|subdir| subdir.join(binary)) - .find(|candidate| candidate.is_file()) -} - /// The backend label for a packaged llama-server, taken from the nearest ancestor /// directory matching a known backend name (e.g. `vulkan`, `rocm-stable`) so it still /// resolves correctly when Lemonade nests the binary under a build-numbered @@ -552,103 +775,83 @@ pub(crate) fn llama_server_backend_label(server: &Path) -> String { .to_owned() } -/// Where the GGUF for `model_ref` is cached, if it is. -/// -/// `model_ref` may be a file path, an `owner/repo[:variant]` checkpoint, the built-in -/// default model, or a name from the Lemonade registry shipped in `runtime_dir` (e.g. -/// `Qwen3-0.6B-GGUF`). The runtime directory is required rather than optional so that -/// no caller can fall back to a lookup that cannot see registered names, which is the -/// state in which every non-default model failed to serve on Linux. -pub(crate) fn direct_llama_model_path( - paths: &AppPaths, - runtime_dir: &Path, - model_ref: &str, -) -> Option { - direct_llama_model_path_in(&hf_cache_roots(paths), runtime_dir, model_ref) -} - -/// The lookup over explicit cache `roots`, so a test is not at the mercy of whichever -/// models the host it runs on has cached. -fn direct_llama_model_path_in( - roots: &[PathBuf], - runtime_dir: &Path, +/// Registered model names are exact keys after known shorthand normalization. +/// Metadata is untrusted: it must name a safe HF checkpoint with an explicit +/// variant, never a local path or a model guessed from the registered name. +pub(crate) fn resolve_direct_llama_checkpoint( + manifest: &LemonadeInstallManifest, model_ref: &str, -) -> Option { - let as_path = PathBuf::from(model_ref); - if as_path.is_file() { - return Some(as_path); - } +) -> Result> { if let Some(checkpoint) = parse_hf_checkpoint(model_ref) { - return roots - .iter() - .find_map(|root| find_hf_checkpoint_gguf(root, &checkpoint)); - } - if resolve_lemonade_model_ref(model_ref) == DEFAULT_MODEL { - return roots.iter().cloned().find_map(find_default_qwen_gguf); - } - // A name Lemonade registers, other than the default: its registry says which - // checkpoint it downloads, and that checkpoint's GGUF is what ends up in the cache. - // Aliases (`qwen-smoke`, `tiny`, ...) name a registered model too. - let checkpoint = registry_checkpoint(runtime_dir, &resolve_lemonade_model_ref(model_ref))?; - let mut matches = hf_checkpoint_gguf_matches_in(roots, &checkpoint); - // Lemonade fetches the one file its registry names, so a single match is the - // expected case. Several (a `Q4_0` label also naming an `IQ4_0` file, say) cannot - // be told apart from here, and serving a different quantization than the one - // registered is worse than not finding the file. - if matches.len() == 1 { - matches.pop() - } else { - None + return require_checkpoint_variant(checkpoint, model_ref).map(Some); } -} - -/// Why direct ROCm serving cannot find a GGUF for `model_ref` after Lemonade loaded it, -/// worded from the same lookup that failed so the text cannot disagree with it: several -/// cached files matching one registered checkpoint is a different problem, with a -/// different remedy, from none being found. -fn unresolved_model_message(roots: &[PathBuf], runtime_dir: &Path, model_ref: &str) -> String { - if let Some(checkpoint) = - registry_checkpoint(runtime_dir, &resolve_lemonade_model_ref(model_ref)) + let registry_path = manifest.runtime_dir.join("resources/server_models.json"); + let Ok(bytes) = fs::read(®istry_path) else { + return Ok(None); + }; + let Ok(registry) = serde_json::from_slice::(&bytes) else { + return Ok(None); + }; + let Some(entries) = registry.as_object() else { + return Ok(None); + }; + let name = resolve_lemonade_model_ref(model_ref); + let Some(metadata) = entries.get(&name) else { + return Ok(None); + }; + if metadata.get("recipe").and_then(serde_json::Value::as_str) != Some(crate::LLAMACPP_RECIPE) { + return Ok(None); + } + let Some(checkpoint_ref) = metadata + .get("checkpoint") + .and_then(serde_json::Value::as_str) + else { + return Ok(None); + }; + // A registered checkpoint that is not a plain `owner/repo:variant` is not + // this module's to serve: Lemonade's catalog writes `*` for "every file", for + // one. Nothing here builds a path from it, so it goes to the router, which + // served it before. Only a reference the user typed is an error (above). + let Some(checkpoint) = parse_hf_checkpoint(checkpoint_ref) else { + return Ok(None); + }; + let Some(fields) = metadata.as_object() else { + return Ok(None); + }; + // Only the single-checkpoint shape is supported directly. Extra files, + // multimodal inputs, options and unknown fields belong to Lemonade's router. + // In the 11.5.1 source, size/suggested describe catalog entries and these + // labels classify chat models. Other labels can change launch flags (MTP). + if checkpoint.variant.is_none() + || fields.iter().any(|(key, value)| match key.as_str() { + "recipe" | "checkpoint" => false, + "size" => !value.as_f64().is_some_and(|size| size >= 0.0), + "suggested" => !value.is_boolean(), + "labels" => !value.as_array().is_some_and(|labels| { + labels.iter().all(|label| { + matches!(label.as_str(), Some("reasoning" | "tool-calling" | "tools")) + }) + }), + _ => true, + }) { - let matches = hf_checkpoint_gguf_matches_in(roots, &checkpoint); - if matches.len() > 1 { - let names = matches - .iter() - .filter_map(|path| path.file_name().and_then(|name| name.to_str())) - .collect::>() - .join(", "); - return format!( - "`{model_ref}` is registered as `{}/{}` and {} cached files match it ({names}), so direct ROCm serving cannot tell which to use; serve the one you want by its exact file name as `{}/{}:.gguf`", - checkpoint.owner, - checkpoint.repo, - matches.len(), - checkpoint.owner, - checkpoint.repo, - ); - } + return Ok(None); } - format!( - "no cached GGUF was found for `{model_ref}`, so direct ROCm serving cannot start (searched {})", - render_hf_cache_roots(roots) - ) + Ok(Some(checkpoint)) } -/// Lemonade's model registry inside the extracted embeddable, mapping each registered -/// model name to the Hugging Face checkpoint it downloads. -const SERVER_MODELS_RESOURCE: &str = "resources/server_models.json"; +fn require_checkpoint_variant(checkpoint: HfCheckpoint, model_ref: &str) -> Result { + if let Some(message) = hf_checkpoint_variant_required(model_ref) { + bail!("{message}"); + } + Ok(checkpoint) +} -/// The Hugging Face checkpoint Lemonade's registry in `runtime_dir` records for the -/// registered model `name` (e.g. `Qwen3-0.6B-GGUF` -> `unsloth/Qwen3-0.6B-GGUF:Q4_0`), -/// or `None` when the registry is absent or unreadable, does not list `name`, or -/// records something that is not a safe `owner/repo[:variant]` checkpoint. -/// -/// The checkpoint is validated by [`parse_hf_checkpoint`], which is what keeps a -/// hand-edited registry from steering the cache lookup outside the cache. -fn registry_checkpoint(runtime_dir: &Path, name: &str) -> Option { - let registry = fs::read_to_string(runtime_dir.join(SERVER_MODELS_RESOURCE)).ok()?; - let registry = serde_json::from_str::(®istry).ok()?; - let checkpoint = registry.get(name)?.get("checkpoint")?.as_str()?; - parse_hf_checkpoint(checkpoint) +pub(crate) fn direct_serve_binary(manifest: &LemonadeInstallManifest) -> Option { + if manifest.backend_recipe != crate::LLAMACPP_RECIPE { + return None; + } + find_llama_server_binary_for_backend(manifest, &manifest.backend_name) } fn hf_cache_roots(paths: &AppPaths) -> Vec { @@ -711,24 +914,6 @@ fn push_hf_cache_root(roots: &mut Vec, path: PathBuf) { roots.push(path); } -fn find_default_qwen_gguf(cache_root: PathBuf) -> Option { - let snapshots = cache_root.join(DEFAULT_MODEL_REPO_DIR).join("snapshots"); - let entries = fs::read_dir(snapshots).ok()?; - entries - .flatten() - .map(|entry| entry.path().join(DEFAULT_MODEL_GGUF)) - .find(|path| path.is_file()) -} - -/// Locate the downloaded GGUF for a Hugging Face checkpoint inside one hub cache root, -/// selecting the file that matches the requested `:variant` (if any). -fn find_hf_checkpoint_gguf(cache_root: &Path, checkpoint: &HfCheckpoint) -> Option { - select_gguf_for_variant( - collect_hf_checkpoint_ggufs(cache_root, checkpoint), - checkpoint.variant.as_deref(), - ) -} - /// All cached GGUF files for a checkpoint's repo under one hub cache root (unfiltered /// by variant). fn collect_hf_checkpoint_ggufs(cache_root: &Path, checkpoint: &HfCheckpoint) -> Vec { @@ -747,24 +932,111 @@ fn collect_hf_checkpoint_ggufs(cache_root: &Path, checkpoint: &HfCheckpoint) -> /// deduplicated by file name. Used to detect an ambiguous variant (more than one match) /// so the caller can refuse to pick arbitrarily. fn hf_checkpoint_gguf_matches(paths: &AppPaths, checkpoint: &HfCheckpoint) -> Vec { - hf_checkpoint_gguf_matches_in(&hf_cache_roots(paths), checkpoint) + hf_checkpoint_gguf_matches_in_roots(&hf_cache_roots(paths), checkpoint) } -/// [`hf_checkpoint_gguf_matches`] over explicit cache `roots`. -fn hf_checkpoint_gguf_matches_in(roots: &[PathBuf], checkpoint: &HfCheckpoint) -> Vec { - let mut matches = Vec::new(); - let mut seen_names = std::collections::HashSet::new(); - for root in roots { - let ggufs = collect_hf_checkpoint_ggufs(root, checkpoint); - for gguf in filter_ggufs_by_variant(ggufs, checkpoint.variant.as_deref()) { - if let Some(name) = gguf.file_name().and_then(|name| name.to_str()) - && seen_names.insert(name.to_owned()) - { - matches.push(gguf); - } +fn hf_checkpoint_gguf_matches_in_roots( + roots: &[PathBuf], + checkpoint: &HfCheckpoint, +) -> Vec { + // Keep root priority and path order, but prefer a complete split model over + // partial copies of the same files. Deduplicating first could combine shards + // from two revisions, hiding a usable complete download behind an older one. + let mut candidates = roots + .iter() + .flat_map(|root| { + filter_ggufs_by_variant( + collect_hf_checkpoint_ggufs(root, checkpoint), + checkpoint.variant.as_deref(), + ) + }) + .collect::>(); + let mut groups = std::collections::HashMap::new(); + for path in &candidates { + if let Some((stem, _, total)) = parse_gguf_shard(path) { + groups + .entry((path.parent().map(Path::to_path_buf), stem, total)) + .or_insert_with(Vec::new) + .push(path.clone()); } } - matches + let complete = groups + .into_iter() + .filter_map(|(key, paths)| first_shard_of_complete_set(&paths).map(|_| key)) + .collect::>(); + candidates.sort_by_key(|path| { + parse_gguf_shard(path).is_some_and(|(stem, _, total)| { + !complete.contains(&(path.parent().map(Path::to_path_buf), stem, total)) + }) + }); + let mut seen_names = std::collections::HashSet::new(); + candidates.retain(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| seen_names.insert(name.to_owned())) + }); + candidates +} + +/// Split a split-GGUF file name (`-00001-of-00002.gguf`) into its stem, +/// 1-based shard number and shard count. +fn parse_gguf_shard(path: &Path) -> Option<(String, u32, u32)> { + let name = path.file_name()?.to_str()?; + if !name.to_ascii_lowercase().ends_with(".gguf") { + return None; + } + // The suffix is ASCII, so this slice is on a character boundary. + let stem = &name[..name.len() - ".gguf".len()]; + let (rest, total) = stem.rsplit_once("-of-")?; + let (stem, index) = rest.rsplit_once('-')?; + let digits = |text: &str| text.len() == 5 && text.chars().all(|c| c.is_ascii_digit()); + if stem.is_empty() || !digits(index) || !digits(total) { + return None; + } + Some((stem.to_owned(), index.parse().ok()?, total.parse().ok()?)) +} + +/// The first shard when `matches` is exactly one complete split model (every shard +/// of one stem present once), `None` otherwise. Several unrelated files, or an +/// incomplete set, are still ambiguous or missing and are handled by the caller. +fn first_shard_of_complete_set(matches: &[PathBuf]) -> Option { + let shards = matches + .iter() + .map(|path| parse_gguf_shard(path).map(|shard| (path, shard))) + .collect::>>()?; + let (first_path, (stem, _, total)) = shards.first()?; + // llama-server finds the remaining shards beside the first one, so shards + // spread over several cache roots or snapshot revisions are not a usable set. + let directory = first_path.parent(); + let complete = shards.len() == usize::try_from(*total).ok()? + && shards.iter().all(|(path, (other, _, other_total))| { + other == stem && other_total == total && path.parent() == directory + }) + && (1..=*total).all(|wanted| shards.iter().any(|(_, (_, index, _))| *index == wanted)); + if !complete { + return None; + } + shards + .iter() + .find(|(_, (_, index, _))| *index == 1) + .map(|(path, _)| (*path).clone()) +} + +/// Whether `matches` are only shards of a single split model, with some missing. +fn is_incomplete_shard_set(matches: &[PathBuf]) -> bool { + let Some(shards) = matches + .iter() + .map(|path| parse_gguf_shard(path)) + .collect::>>() + else { + return false; + }; + let Some((stem, _, total)) = shards.first() else { + return false; + }; + shards + .iter() + .all(|(other, _, other_total)| other == stem && other_total == total) } /// Collect `*.gguf` files under a snapshot revision directory. GGUFs usually sit at the @@ -780,23 +1052,17 @@ fn collect_gguf_files(dir: &Path, out: &mut Vec, depth: usize) { let path = entry.path(); if path.is_dir() { collect_gguf_files(&path, out, depth + 1); - } else if path - .extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf")) + } else if path.is_file() + && path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf")) { out.push(path); } } } -/// Pick a single GGUF for a requested variant (the first, in sorted order, of the -/// matches). Sorting is by path — deterministic but not by mtime — which is fine -/// because the caller rejects an ambiguous variant before relying on this pick. -fn select_gguf_for_variant(ggufs: Vec, variant: Option<&str>) -> Option { - filter_ggufs_by_variant(ggufs, variant).into_iter().next() -} - /// All GGUFs matching a requested variant, sorted by path. With no variant, every file /// matches. With a variant, a case-insensitive exact filename match (`:model.gguf`) /// wins outright; otherwise every file whose name contains the quantization label @@ -812,13 +1078,51 @@ fn filter_ggufs_by_variant(mut ggufs: Vec, variant: Option<&str>) -> Ve .and_then(|name| name.to_str()) .map(str::to_owned) }; - if let Some(exact) = ggufs + let exact_matches = ggufs .iter() - .find(|path| file_name(path).is_some_and(|name| name.eq_ignore_ascii_case(variant))) - { - return vec![exact.clone()]; + .filter(|path| file_name(path).is_some_and(|name| name.eq_ignore_ascii_case(variant))) + .collect::>(); + if !exact_matches.is_empty() { + // A shard names its whole split model: serving needs every shard, and the + // ambiguity error suggests a shard's exact filename. The same shard name + // can exist in several snapshot revisions; prefer one with the full set. + let shard_set = |exact: &PathBuf| -> Option> { + let (stem, _, total) = parse_gguf_shard(exact)?; + Some( + ggufs + .iter() + .filter(|path| { + path.parent() == exact.parent() + && parse_gguf_shard(path).is_some_and(|(other, _, other_total)| { + other == stem && other_total == total + }) + }) + .cloned() + .collect(), + ) + }; + let sets = exact_matches + .iter() + .filter_map(|exact| shard_set(exact)) + .collect::>(); + if let Some(set) = sets + .iter() + .find(|set| first_shard_of_complete_set(set).is_some()) + .or_else(|| sets.first()) + { + return set.clone(); + } + return vec![exact_matches[0].clone()]; } let variant_lower = variant.to_ascii_lowercase(); + // An explicit filename is not a quantization substring. If absent, download + // that exact file rather than serve another file with the same suffix. + if Path::new(variant) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("gguf")) + { + return Vec::new(); + } ggufs .into_iter() .filter(|path| { @@ -830,10 +1134,645 @@ fn filter_ggufs_by_variant(mut ggufs: Vec, variant: Option<&str>) -> Ve #[cfg(test)] mod tests { use super::*; - use crate::LLAMACPP_RECIPE; + use crate::install::platform_binary_name; + use crate::process::models_payload_has_ready_model; use crate::test_support::scratch_dir; + use crate::{DEFAULT_MODEL, LLAMACPP_RECIPE}; + use rocm_engine_protocol::EngineRecipeHint; use std::fs; + fn write_registry(manifest: &LemonadeInstallManifest, registry: serde_json::Value) { + let resources = manifest.runtime_dir.join("resources"); + fs::create_dir_all(&resources).unwrap(); + fs::write( + resources.join("server_models.json"), + serde_json::to_vec(®istry).unwrap(), + ) + .unwrap(); + } + + fn serve_request(model_ref: &str) -> ServeHttpRequest { + ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: model_ref.to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![1], + runtime_id: None, + env_id: None, + state_path: PathBuf::from("state.json"), + log_path: None, + engine_recipe: None, + } + } + + #[test] + fn serving_route_covers_platform_recipe_binary_and_policy_boundaries() { + let dir = scratch_dir("serving-routes"); + let mut manifest = test_manifest(dir.join("runtime")); + manifest.backend_name = "vulkan".to_owned(); + write_registry( + &manifest, + serde_json::json!({ + "Qwen3-0.6B-GGUF": {"recipe": "llamacpp", "checkpoint": "owner/repo:Q4_0", "suggested": true, "size": 0.38, "labels": ["reasoning"]}, + "Complex": {"recipe": "llamacpp", "checkpoint": "owner/repo:Q4_0", "mmproj": "owner/repo:mmproj.gguf"}, + "OtherRecipe": {"recipe": "oga", "checkpoint": "owner/repo"} + }), + ); + let binary = manifest + .runtime_dir + .join("bin/llamacpp/vulkan") + .join(platform_binary_name("llama-server")); + fs::create_dir_all(binary.parent().unwrap()).unwrap(); + fs::write(&binary, b"x").unwrap(); + for is_linux in [true, false] { + let requested = serve_request("qwen-smoke"); + let route = serving_route(&manifest, &requested, is_linux).unwrap(); + if is_linux { + let ServeRoute::Direct { server, .. } = route else { + panic!("a registered alias with a selected-backend binary serves directly"); + }; + assert_eq!(server, binary); + // The `--alias` the server is started with is asserted through the + // production serve path in + // `direct_serve_through_serve_http_uses_the_requested_alias` (lib.rs). + } else { + assert!( + matches!(route, ServeRoute::Router { model_ref } if model_ref == "Qwen3-0.6B-GGUF") + ); + } + assert!(matches!( + serving_route(&manifest, &serve_request("owner/repo:Q4_0"), is_linux).unwrap(), + ServeRoute::Direct { .. } + )); + assert!(serving_route(&manifest, &serve_request("owner/repo"), is_linux).is_err()); + for name in ["Complex", "Unknown", "OtherRecipe"] { + assert!( + matches!(serving_route(&manifest, &serve_request(name), is_linux).unwrap(), ServeRoute::Router { model_ref } if model_ref == name) + ); + } + let mut cpu_request = serve_request("qwen-smoke"); + cpu_request.device_policy = rocm_engine_protocol::DevicePolicy::CpuOnly; + assert!( + serving_route(&manifest, &cpu_request, is_linux) + .unwrap_err() + .to_string() + .contains("no CPU fallback") + ); + } + fs::remove_file(&binary).unwrap(); + for is_linux in [true, false] { + assert!(matches!( + serving_route(&manifest, &serve_request("qwen-smoke"), is_linux).unwrap(), + ServeRoute::Router { .. } + )); + // The selected backend is supported but its binary is gone: the error is + // returned (no route, so nothing is started) and names the command that + // restores it. `rocm engines install lemonade --reinstall` is a real + // invocation: the `engines install` subcommand takes `--reinstall`, which + // re-extracts the runtime and reinstalls the best backend. + let error = serving_route(&manifest, &serve_request("owner/repo:Q4_0"), is_linux) + .unwrap_err() + .to_string(); + assert!(error.contains("no selected GPU llama-server"), "{error}"); + // This pins the wording only. That the command exists and takes `--reinstall` + // is exercised by `@id:runtime-lemonade-backend-alignment-reported` and + // `@id:runtime-lemonade-backend-alignment-opt-out` (runtime_setup.feature), + // which run `rocm engines install lemonade --reinstall` and require it to + // succeed. + // That first scenario (runtime-16) deletes the selected backend's + // llama-server, reinstalls, and requires the file to be back, using the + // same layout as `direct_serve_binary`. + // That a reinstall re-extracts the runtime over an existing one is + // `a_version_change_forces_re_extraction_over_the_existing_runtime` + // (install.rs). + assert!( + error.contains("rocm engines install lemonade --reinstall"), + "{error}" + ); + } + // Recipe/backend refusal is independent of a missing packaged binary. + fs::write(&binary, b"x").unwrap(); + let cpu_binary = manifest + .runtime_dir + .join("bin/llamacpp/cpu") + .join(platform_binary_name("llama-server")); + fs::create_dir_all(cpu_binary.parent().unwrap()).unwrap(); + fs::write(cpu_binary, b"x").unwrap(); + for (recipe, backend) in [ + ("oga", "vulkan"), + (LLAMACPP_RECIPE, "cpu"), + (LLAMACPP_RECIPE, "unknown"), + ] { + manifest.backend_recipe = recipe.to_owned(); + manifest.backend_name = backend.to_owned(); + for is_linux in [true, false] { + assert!(matches!( + serving_route(&manifest, &serve_request("qwen-smoke"), is_linux).unwrap(), + ServeRoute::Router { .. } + )); + let error = serving_route(&manifest, &serve_request("owner/repo:Q4_0"), is_linux) + .unwrap_err() + .to_string(); + let expected = if recipe == LLAMACPP_RECIPE { + "does not support GPU direct serving" + } else { + "does not support direct HF checkpoint serving" + }; + assert!(error.contains(expected), "{error}"); + // A reinstall cannot make a CPU or non-llama.cpp backend servable, and + // their binary may well be present, so neither claims it is missing. + assert!(!error.contains("is installed under"), "{error}"); + assert!(!error.contains("--reinstall"), "{error}"); + } + } + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_variant_errors_suggest_only_what_was_seen() { + let dir = scratch_dir("variant-examples"); + let manifest = test_manifest(dir.join("runtime")); + let error = resolve_direct_llama_model(&manifest, "owner/repo") + .unwrap_err() + .to_string(); + assert!(error.contains("`owner/repo` needs an explicit"), "{error}"); + // No quantization is suggested for a repo nothing has looked inside. + assert!(error.contains("`owner/repo:`"), "{error}"); + assert!(!error.contains("Q4_K_M"), "{error}"); + // The same failure through the serve entry point returns the error and + // no route, so nothing is downloaded or started. + let error = serving_route(&manifest, &serve_request("owner/repo"), true) + .unwrap_err() + .to_string(); + assert!(error.contains("`owner/repo:`"), "{error}"); + assert!(!error.contains("Q4_K_M"), "{error}"); + + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let snapshot = paths + .cache_dir + .join("huggingface/hub/models--owner--repo/snapshots/revision"); + fs::create_dir_all(&snapshot).unwrap(); + for file in ["model-Q4_0.gguf", "model-Q4_K_M.gguf"] { + fs::write(snapshot.join(file), b"x").unwrap(); + } + let ambiguous = parse_hf_checkpoint("owner/repo:Q4").unwrap(); + let error = direct_llama_model_path(&paths, &DirectLlamaModel::Checkpoint(ambiguous)) + .unwrap_err() + .to_string(); + assert!(error.contains("matches multiple files"), "{error}"); + // The example is a real, unambiguous reference: it resolves to one file. + assert!( + error.contains("e.g. `owner/repo:model-Q4_0.gguf`"), + "{error}" + ); + let example = parse_hf_checkpoint("owner/repo:model-Q4_0.gguf").unwrap(); + assert!( + direct_llama_model_path(&paths, &DirectLlamaModel::Checkpoint(example)) + .unwrap() + .is_some() + ); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_regression_complex_registered_models_use_router() { + let dir = scratch_dir("registered-complex"); + let manifest = test_manifest(dir.join("runtime")); + for (key, value) in [ + ("mmproj", serde_json::json!("owner/repo:mmproj.gguf")), + ( + "checkpoints", + serde_json::json!(["owner/repo:Q4_0", "owner/repo:mmproj.gguf"]), + ), + ("extra_files", serde_json::json!(["tokenizer.json"])), + ("options", serde_json::json!({"ctx_size": 8192})), + ("source", serde_json::json!("alternate")), + ("unknown_option", serde_json::json!(true)), + ("recipe_options", serde_json::json!({"ctx_size": 8192})), + ("labels", serde_json::json!(["vision"])), + ("labels", serde_json::json!(["mtp"])), + ("labels", serde_json::json!(["unknown"])), + ("labels", serde_json::json!([123])), + ("labels", serde_json::json!("reasoning")), + ("size", serde_json::json!("0.38")), + ("suggested", serde_json::json!(1)), + ] { + let mut metadata = + serde_json::json!({"recipe": "llamacpp", "checkpoint": "owner/repo:Q4_0"}); + metadata[key] = value; + write_registry(&manifest, serde_json::json!({"Complex-GGUF": metadata})); + assert!( + resolve_direct_llama_model(&manifest, "Complex-GGUF") + .unwrap() + .is_none(), + "{key}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + /// The chat-model labels the catalog uses do not change how a model is + /// launched, so an entry carrying only those still serves directly. Each label + /// on its own, so dropping one from the allow-list turns this red. + #[test] + fn simple_registered_models_with_chat_labels_serve_directly() { + let dir = scratch_dir("registered-labels"); + let manifest = test_manifest(dir.join("runtime")); + for label in ["reasoning", "tool-calling", "tools"] { + write_registry( + &manifest, + serde_json::json!({"Labelled-GGUF": { + "recipe": "llamacpp", + "checkpoint": "owner/repo:Q4_0", + "labels": [label], + }}), + ); + assert!( + matches!( + resolve_direct_llama_model(&manifest, "Labelled-GGUF").unwrap(), + Some(DirectLlamaModel::Checkpoint(_)) + ), + "{label}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_regression_safe_variantless_registered_model_uses_router() { + let dir = scratch_dir("registered-variantless"); + let manifest = test_manifest(dir.join("runtime")); + write_registry( + &manifest, + serde_json::json!({ + "Variantless": {"recipe": "llamacpp", "checkpoint": "owner/repo"} + }), + ); + assert!( + resolve_direct_llama_model(&manifest, "Variantless") + .unwrap() + .is_none() + ); + assert!(resolve_direct_llama_model(&manifest, "owner/repo").is_err()); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_regression_unusable_catalog_uses_router() { + let dir = scratch_dir("registered-unusable"); + let manifest = test_manifest(dir.join("runtime")); + let catalog = manifest.runtime_dir.join("resources/server_models.json"); + fs::create_dir_all(catalog.parent().unwrap()).unwrap(); + for bytes in [b"[]".as_slice(), b"null".as_slice(), b"invalid".as_slice()] { + fs::write(&catalog, bytes).unwrap(); + assert!( + resolve_direct_llama_model(&manifest, "qwen-smoke") + .unwrap() + .is_none() + ); + } + fs::remove_file(&catalog).unwrap(); + fs::create_dir(&catalog).unwrap(); + assert!( + resolve_direct_llama_model(&manifest, "qwen-smoke") + .unwrap() + .is_none() + ); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_regression_registered_alias_resolves_exact_checkpoint() { + let dir = scratch_dir("registered-alias"); + let manifest = test_manifest(dir.join("runtime")); + write_registry( + &manifest, + serde_json::json!({ + "Qwen3-0.6B-GGUF": {"recipe": "llamacpp", "checkpoint": "unsloth/Qwen3-0.6B-GGUF:Q4_0"}, + "Other-GGUF": {"recipe": "llamacpp", "checkpoint": "owner/Other-GGUF:Q8_0"} + }), + ); + for alias in ["qwen-smoke", "tiny", " QWEN3-0.6B-GGUF "] { + assert_eq!( + resolve_direct_llama_checkpoint(&manifest, alias).unwrap(), + parse_hf_checkpoint("unsloth/Qwen3-0.6B-GGUF:Q4_0") + ); + } + assert_eq!( + resolve_direct_llama_checkpoint(&manifest, "Other-GGUF").unwrap(), + parse_hf_checkpoint("owner/Other-GGUF:Q8_0") + ); + assert_eq!( + resolve_direct_llama_checkpoint(&manifest, "Other").unwrap(), + None + ); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_regression_routes_unsafe_registered_metadata_to_the_router() { + let dir = scratch_dir("registered-unsafe"); + let manifest = test_manifest(dir.join("runtime")); + for checkpoint in [ + "../../etc/passwd:Q4_0", + "owner/repo:../secret", + "/tmp/model.gguf", + ] { + write_registry( + &manifest, + serde_json::json!({ + "Unsafe-GGUF": {"recipe": "llamacpp", "checkpoint": checkpoint} + }), + ); + // Never served directly, and never turned into a path: the router + // gets the name, as it did before direct serving existed. + assert_eq!( + resolve_direct_llama_checkpoint(&manifest, "Unsafe-GGUF").unwrap(), + None, + "{checkpoint}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + /// Lemonade's own catalog writes `*` for "every file". It is not a safe + /// filename, so direct serving cannot take it, but it is not malformed either: + /// the serve must go to the router, and the fallback must agree. + #[test] + fn a_registered_wildcard_variant_routes_to_the_router_not_an_error() { + let dir = scratch_dir("registered-wildcard"); + let mut manifest = test_manifest(dir.join("runtime")); + manifest.backend_name = "vulkan".to_owned(); + write_registry( + &manifest, + serde_json::json!({ + "Wildcard-GGUF": { + "checkpoint": "owner/repo:*", + "recipe": "llamacpp", + "suggested": true, + "labels": ["reasoning"], + "size": 1.0 + } + }), + ); + let binary = manifest + .runtime_dir + .join("bin/llamacpp/vulkan") + .join(platform_binary_name("llama-server")); + fs::create_dir_all(binary.parent().unwrap()).unwrap(); + fs::write(&binary, b"x").unwrap(); + for is_linux in [true, false] { + assert!( + matches!( + serving_route(&manifest, &serve_request("Wildcard-GGUF"), is_linux).unwrap(), + ServeRoute::Router { ref model_ref } if model_ref == "Wildcard-GGUF" + ), + "is_linux {is_linux}" + ); + assert!( + direct_fallback_route(&manifest, &serve_request("Wildcard-GGUF"), is_linux) + .is_none(), + "is_linux {is_linux}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + /// `size` is part of the single-checkpoint shape: a negative one is not a + /// catalog entry this module understands, and zero is. + #[test] + fn a_registered_entry_with_a_negative_size_stays_with_the_router() { + let dir = scratch_dir("registered-size"); + let manifest = test_manifest(dir.join("runtime")); + for (size, direct) in [(-1.0, false), (0.0, true), (0.38, true)] { + write_registry( + &manifest, + serde_json::json!({ + "Sized-GGUF": { + "checkpoint": "owner/repo:Q4_0", + "recipe": "llamacpp", + "size": size + } + }), + ); + assert_eq!( + resolve_direct_llama_checkpoint(&manifest, "Sized-GGUF") + .unwrap() + .is_some(), + direct, + "size {size}" + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + /// Two different split models with the same shard count are not one set, + /// whatever their indexes: serving the first would leave llama-server looking + /// for shards of a model that is not there. + #[test] + fn shards_of_two_different_models_are_not_a_complete_set() { + let dir = Path::new("/snapshots/rev"); + let paths = |names: [&str; 2]| names.map(|name| dir.join(name)).to_vec(); + assert_eq!( + first_shard_of_complete_set(&paths(["a-00001-of-00002.gguf", "b-00002-of-00002.gguf"])), + None + ); + assert_eq!( + first_shard_of_complete_set(&paths(["a-00002-of-00002.gguf", "a-00001-of-00002.gguf"])), + Some(dir.join("a-00001-of-00002.gguf")) + ); + } + + /// A variant that names a `.gguf` file is an exact filename, not a quantization + /// label: when no file has that name, nothing matches, even though another + /// file's name ends with the same text. + #[test] + fn an_exact_gguf_variant_that_is_absent_does_not_match_a_longer_file_name() { + let ggufs = vec![PathBuf::from("/snapshots/rev/model-Q4_0.gguf")]; + assert_eq!( + filter_ggufs_by_variant(ggufs.clone(), Some("Q4_0.gguf")), + Vec::::new() + ); + // The same text as a label still matches by substring. + assert_eq!(filter_ggufs_by_variant(ggufs.clone(), Some("Q4_0")), ggufs); + } + + #[test] + fn serving_regression_selected_backend_ignores_stale_rocm() { + let dir = scratch_dir("selected-backend"); + let mut manifest = test_manifest(dir.join("runtime")); + manifest.backend_name = "vulkan".to_owned(); + let base = manifest.runtime_dir.join("bin/llamacpp"); + for backend in ["rocm-stable", "vulkan", "cpu"] { + fs::create_dir_all(base.join(backend)).unwrap(); + fs::write( + base.join(backend) + .join(platform_binary_name("llama-server")), + b"x", + ) + .unwrap(); + } + assert_eq!( + direct_serve_binary(&manifest), + Some( + base.join("vulkan") + .join(platform_binary_name("llama-server")) + ) + ); + fs::remove_dir_all(base.join("vulkan")).unwrap(); + assert_eq!(direct_serve_binary(&manifest), None); + manifest.backend_name = "cpu".to_owned(); + assert_eq!(direct_serve_binary(&manifest), None); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_registered_recipe_boundaries_and_missing_registry() { + let dir = scratch_dir("registry-boundaries"); + let manifest = test_manifest(dir.join("runtime")); + assert!( + resolve_direct_llama_model(&manifest, "qwen-smoke") + .unwrap() + .is_none() + ); + write_registry( + &manifest, + serde_json::json!({ + "RecipeModel": {"recipe": "oga", "checkpoint": "not/a/gguf"}, + "NoCheckpoint": {"recipe": "llamacpp"}, + "MalformedCheckpoint": {"recipe": "llamacpp", "checkpoint": 123}, + "Exact-GGUF": {"recipe": "llamacpp", "checkpoint": "owner/repo:Q8_0"} + }), + ); + for name in [ + "RecipeModel", + "Unknown-GGUF", + "exact-gguf", + "Exact", + "../../escape", + "NoCheckpoint", + "MalformedCheckpoint", + ] { + assert!( + resolve_direct_llama_model(&manifest, name) + .unwrap() + .is_none(), + "{name}" + ); + } + assert!(resolve_direct_llama_model(&manifest, "owner/repo").is_err()); + for json in [b"[]".as_slice(), b"invalid".as_slice()] { + fs::write( + manifest.runtime_dir.join("resources/server_models.json"), + json, + ) + .unwrap(); + assert!( + resolve_direct_llama_model(&manifest, "Exact-GGUF") + .unwrap() + .is_none() + ); + } + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_registered_checkpoint_cache_rejects_ambiguous_variants() { + let dir = scratch_dir("registry-cache"); + let manifest = test_manifest(dir.join("runtime")); + let roots = vec![dir.join("hub-one"), dir.join("hub-two")]; + for root in &roots { + let snapshot = root.join("models--owner--repo/snapshots/revision"); + fs::create_dir_all(&snapshot).unwrap(); + for file in ["model-Q4_0.gguf", "model-Q4_K_M.gguf"] { + fs::write(snapshot.join(file), b"x").unwrap(); + } + // A GGUF suffix on a directory is not a downloaded model file. This + // fixture is handled by the `is_dir()` branch of `collect_gguf_files`; the + // `is_file()` filter is covered by + // `collect_gguf_files_skips_dangling_symlinks`. + fs::create_dir_all(snapshot.join("directory-Q4.gguf")).unwrap(); + } + write_registry( + &manifest, + serde_json::json!({ + "Exact-GGUF": {"recipe": "llamacpp", "checkpoint": "owner/repo:model-Q4_0.gguf"}, + "Ambiguous-GGUF": {"recipe": "llamacpp", "checkpoint": "owner/repo:Q4"} + }), + ); + let exact = resolve_direct_llama_checkpoint(&manifest, "Exact-GGUF") + .unwrap() + .unwrap(); + let matches = hf_checkpoint_gguf_matches_in_roots(&roots, &exact); + assert_eq!( + matches.len(), + 1, + "same filename across cache roots is one quantization" + ); + assert_eq!(matches[0].file_name().unwrap(), "model-Q4_0.gguf"); + let ambiguous = resolve_direct_llama_checkpoint(&manifest, "Ambiguous-GGUF") + .unwrap() + .unwrap(); + assert_eq!( + hf_checkpoint_gguf_matches_in_roots(&roots, &ambiguous).len(), + 2 + ); + // Path resolution must reject ambiguous matching, not pick sorted first. + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let snapshot = paths + .cache_dir + .join("huggingface/hub/models--owner--repo/snapshots/revision"); + fs::create_dir_all(&snapshot).unwrap(); + for file in ["model-Q4_0.gguf", "model-Q4_K_M.gguf"] { + fs::write(snapshot.join(file), b"x").unwrap(); + } + assert!(direct_llama_model_path(&paths, &DirectLlamaModel::Checkpoint(ambiguous)).is_err()); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_local_gguf_is_explicit_not_registry_path_metadata() { + let dir = scratch_dir("local-gguf"); + let manifest = test_manifest(dir.join("runtime")); + let model = dir.join("model.GGUF"); + fs::write(&model, b"x").unwrap(); + let local = resolve_direct_llama_model(&manifest, model.to_str().unwrap()) + .unwrap() + .unwrap(); + assert!(matches!(local, DirectLlamaModel::Local(ref path) if path == &model)); + write_registry( + &manifest, + serde_json::json!({ + "Path-GGUF": {"recipe": "llamacpp", "checkpoint": model} + }), + ); + // A registry entry never names a local file to serve: it is left to the + // router, not read as a path. + assert!( + resolve_direct_llama_model(&manifest, "Path-GGUF") + .unwrap() + .is_none() + ); + let non_gguf = dir.join("model.txt"); + fs::write(&non_gguf, b"x").unwrap(); + assert!( + resolve_direct_llama_model(&manifest, non_gguf.to_str().unwrap()) + .unwrap() + .is_none() + ); + fs::remove_dir_all(dir).unwrap(); + } + #[test] fn parses_canonical_hugging_face_checkpoints() { assert_eq!( @@ -866,6 +1805,37 @@ mod tests { assert!(parse_hf_checkpoint("owner/repo:").is_none()); } + #[test] + fn checkpoint_matches_prefer_the_highest_priority_cache_root() { + let dir = scratch_dir("root-priority"); + // The higher-priority root sorts after the lower one alphabetically, so a + // merged-then-sorted list would pick the wrong file. + let high = dir.join("zz-high"); + let low = dir.join("aa-low"); + let roots = vec![high.clone(), low.clone()]; + let checkpoint = parse_hf_checkpoint("owner/repo:model-Q4_0.gguf").unwrap(); + let snapshot_file = + |root: &Path| root.join("models--owner--repo/snapshots/revision/model-Q4_0.gguf"); + for root in [&high, &low] { + let file = snapshot_file(root); + fs::create_dir_all(file.parent().unwrap()).unwrap(); + fs::write(&file, b"x").unwrap(); + } + assert_eq!( + hf_checkpoint_gguf_matches_in_roots(&roots, &checkpoint), + vec![snapshot_file(&high)], + "the highest-priority root wins, not the alphabetically first path" + ); + + // A file present only in the lower-priority root is still found. + fs::remove_file(snapshot_file(&high)).unwrap(); + assert_eq!( + hf_checkpoint_gguf_matches_in_roots(&roots, &checkpoint), + vec![snapshot_file(&low)] + ); + fs::remove_dir_all(dir).unwrap(); + } + #[test] fn rejects_path_traversal_in_checkpoint_components() { // A crafted reference must not be able to escape the model cache directory. @@ -877,6 +1847,17 @@ mod tests { assert!(parse_hf_checkpoint("owner/repo:a/b.gguf").is_none()); } + #[test] + fn serving_regression_exact_filename_never_substitutes_suffix_match() { + assert!( + filter_ggufs_by_variant( + vec![PathBuf::from("/cache/other-model.gguf")], + Some("model.gguf") + ) + .is_empty() + ); + } + #[test] fn only_a_variantless_checkpoint_needs_a_variant() { let message = hf_checkpoint_variant_required("owner/repo") @@ -915,18 +1896,29 @@ mod tests { PathBuf::from("/cache/LFM2.5-230M-Q8_0.gguf"), ]; assert_eq!( - select_gguf_for_variant(ggufs.clone(), Some("q4_0")), + filter_ggufs_by_variant(ggufs.clone(), Some("q4_0")) + .into_iter() + .next(), Some(PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf")) ); assert_eq!( - select_gguf_for_variant(ggufs.clone(), Some("LFM2.5-230M-Q8_0.gguf")), + filter_ggufs_by_variant(ggufs.clone(), Some("LFM2.5-230M-Q8_0.gguf")) + .into_iter() + .next(), Some(PathBuf::from("/cache/LFM2.5-230M-Q8_0.gguf")) ); assert_eq!( - select_gguf_for_variant(ggufs.clone(), None), + filter_ggufs_by_variant(ggufs.clone(), None) + .into_iter() + .next(), Some(PathBuf::from("/cache/LFM2.5-230M-Q4_0.gguf")) ); - assert_eq!(select_gguf_for_variant(ggufs, Some("Q2_K")), None); + assert_eq!( + filter_ggufs_by_variant(ggufs, Some("Q2_K")) + .into_iter() + .next(), + None + ); } #[test] @@ -949,7 +1941,7 @@ mod tests { } #[test] - fn find_llama_server_binary_prefers_gpu_and_skips_cpu() { + fn direct_serve_binary_uses_selected_gpu_backend_and_skips_cpu() { let dir = scratch_dir("find-server"); let runtime_dir = dir.join("runtime"); let llamacpp = runtime_dir.join("bin").join("llamacpp"); @@ -958,10 +1950,11 @@ mod tests { fs::create_dir_all(&backend_dir).unwrap(); fs::write(backend_dir.join(platform_binary_name("llama-server")), b"x").unwrap(); } - let manifest = test_manifest(runtime_dir); + let mut manifest = test_manifest(runtime_dir); + manifest.backend_name = "vulkan".to_owned(); // Both cpu and vulkan exist; the GPU backend is chosen and cpu is never selected. assert_eq!( - find_llama_server_binary(&manifest), + direct_serve_binary(&manifest), Some( llamacpp .join("vulkan") @@ -970,12 +1963,12 @@ mod tests { ); // With only cpu present, nothing is returned (no silent CPU fallback). fs::remove_dir_all(llamacpp.join("vulkan")).unwrap(); - assert_eq!(find_llama_server_binary(&manifest), None); + assert_eq!(direct_serve_binary(&manifest), None); fs::remove_dir_all(&dir).ok(); } #[test] - fn find_llama_server_binary_finds_build_numbered_nesting() { + fn direct_serve_binary_finds_build_numbered_nesting() { // Some Lemonade llama.cpp releases extract one level deeper than // `/llama-server`, e.g. `rocm-stable/llama-b9752/llama-server`. let dir = scratch_dir("find-server-nested"); @@ -986,13 +1979,13 @@ mod tests { let server = nested.join(platform_binary_name("llama-server")); fs::write(&server, b"x").unwrap(); let manifest = test_manifest(runtime_dir); - assert_eq!(find_llama_server_binary(&manifest), Some(server.clone())); + assert_eq!(direct_serve_binary(&manifest), Some(server.clone())); assert_eq!(llama_server_backend_label(&server), "rocm-stable"); fs::remove_dir_all(&dir).ok(); } #[test] - fn find_llama_server_binary_follows_backend_priority() { + fn direct_serve_binary_for_rocm_follows_family_priority() { let dir = scratch_dir("find-server-priority"); let runtime_dir = dir.join("runtime"); let llamacpp = runtime_dir.join("bin").join("llamacpp"); @@ -1004,19 +1997,19 @@ mod tests { }; let manifest = test_manifest(runtime_dir); - // Nightly-only host (the resilient case that replaced the hardcoded - // rocm-stable direct-serve path): the nightly backend is selected over vulkan. + // The manifest selects `rocm`. A host whose ROCm build sits in the nightly + // directory still serves from it, and a vulkan install is never used. install("rocm-nightly"); install("vulkan"); assert_eq!( - find_llama_server_binary(&manifest), + direct_serve_binary(&manifest), Some(llamacpp.join("rocm-nightly").join(&server)) ); - // With rocm-stable also present, it wins (highest priority). + // With rocm-stable also present, it wins: it is first in the ROCm family order. install("rocm-stable"); assert_eq!( - find_llama_server_binary(&manifest), + direct_serve_binary(&manifest), Some(llamacpp.join("rocm-stable").join(&server)) ); fs::remove_dir_all(&dir).ok(); @@ -1041,308 +2034,474 @@ mod tests { fs::remove_dir_all(&dir).ok(); } - /// A runtime tree holding a Lemonade registry, and a hub cache holding the GGUFs - /// named in `cached` (`(repo_dir, file)` pairs), under one scratch directory. - fn registry_fixture( - tag: &str, - registry: &str, - cached: &[(&str, &str)], - ) -> (PathBuf, PathBuf, PathBuf) { - let dir = scratch_dir(tag); - let runtime_dir = dir.join("runtime"); - let resources = runtime_dir.join("resources"); - fs::create_dir_all(&resources).unwrap(); - fs::write(resources.join("server_models.json"), registry).unwrap(); - let hub = dir.join("hub"); - for (repo_dir, file) in cached { - let snapshot = hub.join(repo_dir).join("snapshots").join("rev0"); - fs::create_dir_all(&snapshot).unwrap(); - fs::write(snapshot.join(file), b"gguf").unwrap(); - } - (dir, runtime_dir, hub) + /// Hugging Face snapshots hold symlinks into `blobs/`; one whose blob is gone (an + /// interrupted download) is neither a directory nor a file, so only the `is_file()` + /// check keeps it out of the model list. + #[cfg(unix)] + #[test] + fn collect_gguf_files_skips_dangling_symlinks() { + let dir = scratch_dir("collect-gguf-dangling"); + fs::write(dir.join("real-Q4_0.gguf"), b"x").unwrap(); + std::os::unix::fs::symlink(dir.join("missing-blob"), dir.join("dangling-Q8_0.gguf")) + .unwrap(); + assert!(!dir.join("dangling-Q8_0.gguf").is_dir()); + let mut found = Vec::new(); + collect_gguf_files(&dir, &mut found, 0); + assert_eq!(found, vec![dir.join("real-Q4_0.gguf")]); + fs::remove_dir_all(&dir).ok(); } - /// Two entries trimmed from `resources/server_models.json` as shipped in the - /// Lemonade 11.5.1 embeddable (checked against the real file: top-level object keyed - /// by model name, each with a string `checkpoint`). - const SMALL_MODEL_REGISTRY: &str = r#"{ - "Qwen3-0.6B-GGUF": {"checkpoint": "unsloth/Qwen3-0.6B-GGUF:Q4_0", "recipe": "llamacpp"}, - "Qwen3-4B-Instruct-2507-GGUF": { - "checkpoint": "unsloth/Qwen3-4B-Instruct-2507-GGUF:Qwen3-4B-Instruct-2507-Q4_K_M.gguf", - "recipe": "llamacpp" - } - }"#; + #[test] + fn download_uses_validated_checkpoint_ref_not_the_alias_and_returns_the_file() { + let dir = scratch_dir("download-ref"); + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let checkpoint = parse_hf_checkpoint("pull-test-owner/pull-test-repo:Q4_0").unwrap(); + let model = DirectLlamaModel::Checkpoint(checkpoint); + let snapshot = paths + .cache_dir + .join("huggingface/hub/models--pull-test-owner--pull-test-repo/snapshots/revision"); + let mut pulled = Vec::new(); + let path = ensure_model_with_download(&paths, "qwen-smoke", &model, |reference| { + pulled.push(reference.to_owned()); + fs::create_dir_all(&snapshot).unwrap(); + fs::write(snapshot.join("Qwen3-0.6B-Q4_0.gguf"), b"x").unwrap(); + Ok(()) + }) + .unwrap(); + assert_eq!(pulled, ["pull-test-owner/pull-test-repo:Q4_0"]); + assert_eq!(path, snapshot.join("Qwen3-0.6B-Q4_0.gguf")); + // A cached file is served without pulling again. + let path = ensure_model_with_download(&paths, "qwen-smoke", &model, |_| { + panic!("cached model must not be pulled") + }) + .unwrap(); + assert_eq!(path, snapshot.join("Qwen3-0.6B-Q4_0.gguf")); + fs::remove_dir_all(&dir).ok(); + } - /// The failure behind the Linux Lemonade serve lanes: `rocm serve Qwen3-0.6B-GGUF - /// --engine lemonade` loads the model through a short-lived `lemond`, which leaves - /// the GGUF in the hub cache, and the serve then bailed "did not download" because - /// only the built-in default model and `owner/repo:variant` refs could be found. + #[cfg(target_os = "linux")] #[test] - fn a_registered_model_name_resolves_through_its_registry_checkpoint() { - let (dir, runtime_dir, hub) = registry_fixture( - "registered-model", - SMALL_MODEL_REGISTRY, - &[("models--unsloth--Qwen3-0.6B-GGUF", "Qwen3-0.6B-Q4_0.gguf")], - ); - let found = - direct_llama_model_path_in(std::slice::from_ref(&hub), &runtime_dir, "Qwen3-0.6B-GGUF"); + fn real_download_closure_pulls_the_checkpoint_not_the_requested_alias() { + use crate::test_http::fake_runtime; + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + // The fake `lemonade pull` drops the GGUF into the cache, as the real one does. + let manifest = fake_runtime::manifest( + dir, + 0, + Some("models--pull-test-owner--pull-test-repo/snapshots/rev/model-Q4_0.gguf"), + ); + let runtime = LemonadeRuntime { manifest }; + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + // A request whose text is an alias, resolved to a registered checkpoint. + let request = serve_request("qwen-smoke"); + let model = DirectLlamaModel::Checkpoint( + parse_hf_checkpoint("pull-test-owner/pull-test-repo:Q4_0").unwrap(), + ); + let path = ensure_model_in( + &paths, + &request, + &runtime, + &LemonadeProcessEnvironment::default(), + None, + &model, + &ParentRuntimeEnvironment::rooted_at(dir), + ) + .unwrap(); + assert!(path.is_file(), "{}", path.display()); + let pulls = fake_runtime::calls(dir) + .into_iter() + .filter(|call| call.contains(" pull ")) + .collect::>(); + assert_eq!(pulls.len(), 1, "{pulls:?}"); + let argv = pulls[0].split_whitespace().collect::>(); assert_eq!( - found, - Some( - hub.join("models--unsloth--Qwen3-0.6B-GGUF") - .join("snapshots") - .join("rev0") - .join("Qwen3-0.6B-Q4_0.gguf") - ) + argv[argv.len() - 2..], + ["pull", "pull-test-owner/pull-test-repo:Q4_0"], + "{pulls:?}" ); - fs::remove_dir_all(&dir).ok(); + assert!(!pulls[0].contains("qwen-smoke"), "{pulls:?}"); } #[test] - fn a_registered_model_that_is_not_cached_resolves_to_nothing() { - let (dir, runtime_dir, hub) = - registry_fixture("registered-uncached", SMALL_MODEL_REGISTRY, &[]); + fn split_gguf_variant_resolves_to_its_first_shard() { + let dir = scratch_dir("sharded"); + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let model = DirectLlamaModel::Checkpoint( + parse_hf_checkpoint("shard-test-owner/shard-test-repo:Q4_K_M").unwrap(), + ); + let folder = paths + .cache_dir + .join("huggingface/hub/models--shard-test-owner--shard-test-repo/snapshots/rev/Q4_K_M"); + fs::create_dir_all(&folder).unwrap(); + // One shard of two: the set is incomplete, so it counts as not downloaded + // (and is pulled again) rather than being served partial. + fs::write(folder.join("model-Q4_K_M-00002-of-00002.gguf"), b"x").unwrap(); + assert_eq!(direct_llama_model_path(&paths, &model).unwrap(), None); + // Both shards present: one model, served from the first shard. + fs::write(folder.join("model-Q4_K_M-00001-of-00002.gguf"), b"x").unwrap(); assert_eq!( - direct_llama_model_path_in(std::slice::from_ref(&hub), &runtime_dir, "Qwen3-0.6B-GGUF"), - None + direct_llama_model_path(&paths, &model).unwrap(), + Some(folder.join("model-Q4_K_M-00001-of-00002.gguf")) ); + // An unrelated file with the same label makes it ambiguous again. + fs::write(folder.join("other-Q4_K_M.gguf"), b"x").unwrap(); + let error = direct_llama_model_path(&paths, &model) + .unwrap_err() + .to_string(); + assert!(error.contains("matches multiple files"), "{error}"); fs::remove_dir_all(&dir).ok(); } - /// A name the registry does not list is not guessed at, and without a runtime to - /// read a registry from nothing but the old lookups apply. + fn shard_test_paths(dir: &Path) -> AppPaths { + AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + } + } + + fn shard_checkpoint(variant: &str) -> DirectLlamaModel { + DirectLlamaModel::Checkpoint( + parse_hf_checkpoint(&format!("shard-test-owner/shard-test-repo:{variant}")).unwrap(), + ) + } + + fn shard_snapshot(paths: &AppPaths, revision: &str) -> PathBuf { + let folder = paths.cache_dir.join(format!( + "huggingface/hub/models--shard-test-owner--shard-test-repo/snapshots/{revision}" + )); + fs::create_dir_all(&folder).unwrap(); + folder + } + #[test] - fn an_unregistered_name_or_a_missing_registry_resolves_to_nothing() { - let (dir, runtime_dir, hub) = registry_fixture( - "unregistered", - SMALL_MODEL_REGISTRY, - &[("models--unsloth--Qwen3-0.6B-GGUF", "Qwen3-0.6B-Q4_0.gguf")], - ); - let roots = std::slice::from_ref(&hub); + fn split_set_ambiguity_suggestion_resolves_to_the_whole_set() { + let dir = scratch_dir("shard-suggestion"); + let paths = shard_test_paths(&dir); + let folder = shard_snapshot(&paths, "rev"); + for file in [ + "model-Q4_K_M-00001-of-00002.gguf", + "model-Q4_K_M-00002-of-00002.gguf", + "other-Q4_K_M.gguf", + ] { + fs::write(folder.join(file), b"x").unwrap(); + } + let error = direct_llama_model_path(&paths, &shard_checkpoint("Q4_K_M")) + .unwrap_err() + .to_string(); + assert!(error.contains("matches multiple files"), "{error}"); + let suggestion = error + .split_once("e.g. `shard-test-owner/shard-test-repo:") + .and_then(|(_, rest)| rest.split_once('`')) + .map_or_else( + || panic!("no suggestion in {error}"), + |(variant, _)| variant.to_owned(), + ); + assert!(suggestion.contains("-of-"), "{suggestion}"); + // The suggestion, fed back, serves the split model from its first shard. assert_eq!( - direct_llama_model_path_in(roots, &runtime_dir, "Not-A-Registered-Model"), - None + direct_llama_model_path(&paths, &shard_checkpoint(&suggestion)).unwrap(), + Some(folder.join("model-Q4_K_M-00001-of-00002.gguf")) ); - let no_registry = dir.join("no-registry"); - fs::create_dir_all(&no_registry).unwrap(); + // A shard name alone never serves a partial set. + fs::remove_file(folder.join("model-Q4_K_M-00002-of-00002.gguf")).unwrap(); assert_eq!( - direct_llama_model_path_in(roots, &no_registry, "Qwen3-0.6B-GGUF"), + direct_llama_model_path(&paths, &shard_checkpoint(&suggestion)).unwrap(), None ); fs::remove_dir_all(&dir).ok(); } - /// A malformed registry degrades to "not found", never a panic or a guess. #[test] - fn a_malformed_registry_resolves_to_nothing() { - for (tag, registry) in [ - ("registry-not-json", "not json"), - ("registry-not-object", "[]"), - ( - "registry-no-checkpoint", - r#"{"Qwen3-0.6B-GGUF": {"recipe": "llamacpp"}}"#, - ), - ( - "registry-checkpoint-not-string", - r#"{"Qwen3-0.6B-GGUF": {"checkpoint": 7}}"#, - ), - ] { - let (dir, runtime_dir, hub) = registry_fixture( - tag, - registry, - &[("models--unsloth--Qwen3-0.6B-GGUF", "Qwen3-0.6B-Q4_0.gguf")], - ); - assert_eq!( - direct_llama_model_path_in( - std::slice::from_ref(&hub), - &runtime_dir, - "Qwen3-0.6B-GGUF" - ), - None, - "{tag}" - ); - fs::remove_dir_all(&dir).ok(); + fn shards_spread_over_cache_roots_are_not_a_complete_set() { + let dir = scratch_dir("shard-roots"); + let checkpoint = parse_hf_checkpoint("shard-test-owner/shard-test-repo:Q4_K_M").unwrap(); + let mut roots = Vec::new(); + for (name, shard) in [("h1", 1), ("h2", 2)] { + let root = dir.join(name); + let folder = root.join("models--shard-test-owner--shard-test-repo/snapshots/rev"); + fs::create_dir_all(&folder).unwrap(); + fs::write( + folder.join(format!("model-Q4_K_M-0000{shard}-of-00002.gguf")), + b"x", + ) + .unwrap(); + roots.push(root); } + let matches = hf_checkpoint_gguf_matches_in_roots(&roots, &checkpoint); + assert_eq!(matches.len(), 2, "{matches:?}"); + assert_eq!(first_shard_of_complete_set(&matches), None); + assert!(is_incomplete_shard_set(&matches)); + fs::remove_dir_all(&dir).ok(); } - /// The registry is a file inside the runtime tree, so a hand-edited one must not be - /// able to point the cache lookup outside the cache: the checkpoint's components - /// are joined into a path, and `b/../../x` would climb out of the hub through a - /// directory that exists. The target here sits outside the hub and is a real file, - /// so only the validation in `parse_hf_checkpoint` stops it being served. #[test] - fn a_registry_checkpoint_cannot_point_outside_the_cache() { - let (dir, runtime_dir, hub) = registry_fixture( - "registered-escape", - r#"{"Evil-Model": {"checkpoint": "a/b/../../x:Q4_0"}}"#, - &[], - ); - fs::create_dir_all(hub.join("models--a--b")).unwrap(); - let outside = dir.join("x").join("snapshots").join("rev0"); - fs::create_dir_all(&outside).unwrap(); - fs::write(outside.join("escape-Q4_0.gguf"), b"gguf").unwrap(); + fn shards_spread_over_snapshot_revisions_are_not_a_complete_set() { + let dir = scratch_dir("shard-revisions"); + let paths = shard_test_paths(&dir); + for (revision, shard) in [("revA", 1), ("revB", 2)] { + fs::write( + shard_snapshot(&paths, revision) + .join(format!("model-Q4_K_M-0000{shard}-of-00002.gguf")), + b"x", + ) + .unwrap(); + } assert_eq!( - direct_llama_model_path_in(std::slice::from_ref(&hub), &runtime_dir, "Evil-Model"), + direct_llama_model_path(&paths, &shard_checkpoint("Q4_K_M")).unwrap(), None ); + // The same two shards in one revision are served. + let folder = shard_snapshot(&paths, "revA"); + fs::write(folder.join("model-Q4_K_M-00002-of-00002.gguf"), b"x").unwrap(); + assert_eq!( + direct_llama_model_path(&paths, &shard_checkpoint("Q4_K_M")).unwrap(), + Some(folder.join("model-Q4_K_M-00001-of-00002.gguf")) + ); fs::remove_dir_all(&dir).ok(); } - /// The registry's variant is a substring match, so a cache holding two files it - /// matches (`Q4_0` also names `IQ4_0`) must not have one picked arbitrarily. #[test] - fn a_registered_model_matching_two_cached_files_is_not_guessed() { - let (dir, runtime_dir, hub) = registry_fixture( - "registered-ambiguous", - SMALL_MODEL_REGISTRY, - &[ - ("models--unsloth--Qwen3-0.6B-GGUF", "Qwen3-0.6B-Q4_0.gguf"), - ("models--unsloth--Qwen3-0.6B-GGUF", "Qwen3-0.6B-IQ4_0.gguf"), - ], - ); - assert_eq!( - direct_llama_model_path_in(std::slice::from_ref(&hub), &runtime_dir, "Qwen3-0.6B-GGUF"), - None + fn complete_snapshot_is_not_shadowed_by_an_older_partial_download() { + let temporary = tempfile::tempdir().unwrap(); + let paths = shard_test_paths(temporary.path()); + let partial = shard_snapshot(&paths, "aaa"); + let complete = shard_snapshot(&paths, "bbb"); + let first = "model-Q4_K_M-00001-of-00002.gguf"; + fs::write(partial.join(first), b"x").unwrap(); + for file in [first, "model-Q4_K_M-00002-of-00002.gguf"] { + fs::write(complete.join(file), b"x").unwrap(); + } + for variant in ["Q4_K_M", first] { + let model = shard_checkpoint(variant); + let file = ensure_model_with_download(&paths, "req", &model, |_| { + panic!("a complete cached revision must not trigger a pull") + }) + .unwrap(); + assert_eq!(file, complete.join(first)); + } + // A second unrelated model remains ambiguous; preferring a complete + // revision must not turn a broad quantization into an arbitrary choice. + fs::write(complete.join("other-Q4_K_M.gguf"), b"x").unwrap(); + assert!( + direct_llama_model_path(&paths, &shard_checkpoint("Q4_K_M")) + .unwrap_err() + .to_string() + .contains("matches multiple files") ); - fs::remove_dir_all(&dir).ok(); } - /// A registry entry whose variant is an exact filename resolves to that file, which - /// is how the built-in default model is registered. #[test] - fn a_registry_checkpoint_naming_an_exact_file_resolves_to_that_file() { - let (dir, runtime_dir, hub) = registry_fixture( - "registered-exact-file", - SMALL_MODEL_REGISTRY, - &[ - ( - "models--unsloth--Qwen3-4B-Instruct-2507-GGUF", - "Qwen3-4B-Instruct-2507-Q4_K_M.gguf", - ), - ( - "models--unsloth--Qwen3-4B-Instruct-2507-GGUF", - "Qwen3-4B-Instruct-2507-Q8_0.gguf", - ), - ], - ); - // Registered under a name that is not the built-in default's, so only the - // registry lookup can find it. - let registry = - SMALL_MODEL_REGISTRY.replace("Qwen3-4B-Instruct-2507-GGUF\": {", "Other-Name\": {"); - fs::write( - runtime_dir.join("resources").join("server_models.json"), - registry, - ) - .unwrap(); + fn shard_indexes_outside_the_declared_range_are_not_a_complete_set() { + let dir = scratch_dir("shard-range"); + let paths = shard_test_paths(&dir); + let folder = shard_snapshot(&paths, "rev"); + for file in [ + "model-Q4_K_M-00001-of-00002.gguf", + "model-Q4_K_M-00003-of-00002.gguf", + ] { + fs::write(folder.join(file), b"x").unwrap(); + } assert_eq!( - direct_llama_model_path_in(std::slice::from_ref(&hub), &runtime_dir, "Other-Name") - .and_then(|path| path.file_name().map(std::borrow::ToOwned::to_owned)), - Some("Qwen3-4B-Instruct-2507-Q4_K_M.gguf".into()) + direct_llama_model_path(&paths, &shard_checkpoint("Q4_K_M")).unwrap(), + None ); fs::remove_dir_all(&dir).ok(); } - /// The message is derived from the lookup that failed: with two cached files - /// matching one registered checkpoint it names both and how to pick one, and with - /// none it says nothing was found rather than claiming nothing was downloaded. #[test] - fn the_unresolved_message_follows_the_reason_the_lookup_failed() { - let (dir, runtime_dir, hub) = registry_fixture( - "unresolved-ambiguous", - SMALL_MODEL_REGISTRY, - &[ - ("models--unsloth--Qwen3-0.6B-GGUF", "Qwen3-0.6B-Q4_0.gguf"), - ("models--unsloth--Qwen3-0.6B-GGUF", "Qwen3-0.6B-IQ4_0.gguf"), - ], - ); - let roots = std::slice::from_ref(&hub); - // The state: the lookup refuses to pick. + fn shard_parsing_requires_the_split_naming_scheme() { + let parse = |name: &str| parse_gguf_shard(Path::new(name)); assert_eq!( - direct_llama_model_path_in(roots, &runtime_dir, "Qwen3-0.6B-GGUF"), - None - ); - // The message: it says why, and what to do. - let ambiguous = unresolved_model_message(roots, &runtime_dir, "Qwen3-0.6B-GGUF"); - assert!(ambiguous.contains("2 cached files match"), "{ambiguous}"); - assert!( - ambiguous.contains("Qwen3-0.6B-Q4_0.gguf") - && ambiguous.contains("Qwen3-0.6B-IQ4_0.gguf"), - "{ambiguous}" + parse("gpt-oss-120b-Q4_K_M-00001-of-00002.gguf"), + Some(("gpt-oss-120b-Q4_K_M".to_owned(), 1, 2)) ); - assert!( - ambiguous.contains("unsloth/Qwen3-0.6B-GGUF:.gguf"), - "{ambiguous}" - ); - assert!(!ambiguous.contains("did not download"), "{ambiguous}"); + assert_eq!(parse("model-Q4_K_M.gguf"), None); + assert_eq!(parse("model-1-of-2.gguf"), None); + assert_eq!(parse("model-00001-of-00002.bin"), None); + assert_eq!(parse("-00001-of-00002.gguf"), None); + } - // Nothing cached: the lookup finds nothing, and the text does not claim a - // download outcome it cannot know. - let (empty_dir, empty_runtime, empty_hub) = - registry_fixture("unresolved-empty", SMALL_MODEL_REGISTRY, &[]); - let empty_roots = std::slice::from_ref(&empty_hub); - assert_eq!( - direct_llama_model_path_in(empty_roots, &empty_runtime, "Qwen3-0.6B-GGUF"), - None + #[test] + fn download_error_is_returned_and_missing_file_after_pull_is_an_error() { + let dir = scratch_dir("download-error"); + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let model = DirectLlamaModel::Checkpoint( + parse_hf_checkpoint("pull-test-owner/pull-test-repo:model-Q4_0.gguf").unwrap(), ); - let none = unresolved_model_message(empty_roots, &empty_runtime, "Qwen3-0.6B-GGUF"); - assert!(none.contains("no cached GGUF was found"), "{none}"); + let error = ensure_model_with_download(&paths, "alias", &model, |_| { + anyhow::bail!("Lemonade pull of `pull-test-owner/pull-test-repo:model-Q4_0.gguf` failed with status 7") + }) + .unwrap_err() + .to_string(); + assert!(error.contains("failed with status 7"), "{error}"); + // A pull that exits zero without leaving the file is still an error. + let error = ensure_model_with_download(&paths, "alias", &model, |_| Ok(())) + .unwrap_err() + .to_string(); + assert!(error.contains("did not download `alias`"), "{error}"); + // A missing local GGUF is never pulled. + let local = DirectLlamaModel::Local(dir.join("gone.gguf")); + let error = ensure_model_with_download(&paths, "gone.gguf", &local, |_| { + panic!("a local GGUF has nothing to pull") + }) + .unwrap_err() + .to_string(); assert!( - none.contains(&empty_hub.display().to_string()), - "names the searched root: {none}" + error.contains("local GGUF `gone.gguf` is missing"), + "{error}" ); - assert!(!none.contains("did not download"), "{none}"); fs::remove_dir_all(&dir).ok(); - fs::remove_dir_all(&empty_dir).ok(); } - /// `rocm serve qwen-smoke` reaches the lookup under the alias, which Lemonade - /// registers as `Qwen3-0.6B-GGUF`: the registry is keyed by the registered name, so - /// the alias has to be resolved first or only the exact name would ever serve. + /// Real entries from the released 11.5.1 `server_models.json`. Other released + /// entries carry labels such as `hot`, `coding` and `mtp`; those are outside + /// the labels direct serving accepts, so they intentionally route to the router. + fn released_catalog_entries() -> serde_json::Value { + serde_json::json!({ + "Qwen3-4B-Instruct-2507-GGUF": { + "checkpoint": "unsloth/Qwen3-4B-Instruct-2507-GGUF:Qwen3-4B-Instruct-2507-Q4_K_M.gguf", + "recipe": "llamacpp", + "suggested": true, + "labels": ["tool-calling"], + "size": 2.5 + }, + "Qwen3-0.6B-GGUF": { + "checkpoint": "unsloth/Qwen3-0.6B-GGUF:Q4_0", + "recipe": "llamacpp", + "suggested": true, + "labels": ["reasoning"], + "size": 0.38 + }, + "Qwen3.5-0.8B-GGUF": { + "checkpoint": "unsloth/Qwen3.5-0.8B-GGUF:Qwen3.5-0.8B-UD-Q4_K_XL.gguf", + "mmproj": "mmproj-F16.gguf", + "recipe": "llamacpp", + "suggested": true, + "labels": ["vision", "tool-calling"], + "size": 0.764 + }, + "gpt-oss-120b-mxfp-GGUF": { + "checkpoint": "ggml-org/gpt-oss-120b-GGUF:*", + "recipe": "llamacpp", + "suggested": true, + "labels": ["reasoning", "tool-calling"], + "size": 63.0 + } + }) + } + #[test] - fn a_model_alias_resolves_through_the_registry_like_its_registered_name() { - let (dir, runtime_dir, hub) = registry_fixture( - "registered-alias", - SMALL_MODEL_REGISTRY, - &[("models--unsloth--Qwen3-0.6B-GGUF", "Qwen3-0.6B-Q4_0.gguf")], - ); - let roots = std::slice::from_ref(&hub); - let exact = direct_llama_model_path_in(roots, &runtime_dir, "Qwen3-0.6B-GGUF"); - assert!(exact.is_some()); - for alias in ["qwen-smoke", "tiny", "lemonade-tiny", "qwen3-0.6b-gguf"] { - assert_eq!( - direct_llama_model_path_in(roots, &runtime_dir, alias), - exact, - "{alias}" + fn released_catalog_default_model_routes_direct_with_binary_and_router_without() { + let dir = scratch_dir("released-catalog"); + let mut manifest = test_manifest(dir.join("runtime")); + manifest.backend_name = "vulkan".to_owned(); + write_registry(&manifest, released_catalog_entries()); + let binary = manifest + .runtime_dir + .join("bin/llamacpp/vulkan") + .join(platform_binary_name("llama-server")); + + // The engine default is the first released entry. + assert_eq!(DEFAULT_MODEL, "Qwen3-4B-Instruct-2507-GGUF"); + let direct_checkpoints = [ + ( + DEFAULT_MODEL, + "unsloth/Qwen3-4B-Instruct-2507-GGUF:Qwen3-4B-Instruct-2507-Q4_K_M.gguf", + ), + ("Qwen3-0.6B-GGUF", "unsloth/Qwen3-0.6B-GGUF:Q4_0"), + ]; + + // No selected-backend binary: every entry stays on the router. + for name in [DEFAULT_MODEL, "Qwen3-0.6B-GGUF", "Qwen3.5-0.8B-GGUF"] { + assert!( + matches!( + serving_route(&manifest, &serve_request(name), true).unwrap(), + ServeRoute::Router { ref model_ref } if model_ref == name + ), + "{name}" ); } - fs::remove_dir_all(&dir).ok(); + + fs::create_dir_all(binary.parent().unwrap()).unwrap(); + fs::write(&binary, b"x").unwrap(); + for (name, checkpoint) in direct_checkpoints { + let route = serving_route(&manifest, &serve_request(name), true).unwrap(); + let ServeRoute::Direct { model, server } = route else { + panic!("{name} should serve directly with a selected-backend binary"); + }; + assert_eq!(server, binary); + assert!( + matches!(&model, DirectLlamaModel::Checkpoint(found) if found.model_ref() == checkpoint), + "{name}: {model:?}" + ); + } + // The multimodal entry carries `mmproj`, which only the router handles. + assert!(matches!( + serving_route(&manifest, &serve_request("Qwen3.5-0.8B-GGUF"), true).unwrap(), + ServeRoute::Router { model_ref } if model_ref == "Qwen3.5-0.8B-GGUF" + )); + // `ggml-org/gpt-oss-120b-GGUF:*` is how the catalog says "every file". Its + // variant is not a filename, so it is never served directly; with the + // binary present it still goes to the router on Linux and elsewhere, and + // is not an error on either. The fallback after a router failure agrees. + let wildcard = "gpt-oss-120b-mxfp-GGUF"; + for is_linux in [true, false] { + assert!( + matches!( + serving_route(&manifest, &serve_request(wildcard), is_linux).unwrap(), + ServeRoute::Router { ref model_ref } if model_ref == wildcard + ), + "is_linux {is_linux}" + ); + assert!( + direct_fallback_route(&manifest, &serve_request(wildcard), is_linux).is_none(), + "is_linux {is_linux}" + ); + } + fs::remove_dir_all(dir).unwrap(); } - /// `direct_llama_model_path` is the entry point every serve path goes through, so it - /// must pass the runtime's registry on to the lookup: found here only through the - /// registry, in the cache directory the CLI itself owns. #[test] - fn the_entry_point_threads_the_runtime_registry_through() { - let (dir, runtime_dir, _hub) = registry_fixture("entry-point", SMALL_MODEL_REGISTRY, &[]); - let cache_dir = dir.join("cli-cache"); - let snapshot = cache_dir - .join("huggingface") - .join("hub") - .join("models--unsloth--Qwen3-0.6B-GGUF") - .join("snapshots") - .join("rev0"); - fs::create_dir_all(&snapshot).unwrap(); - fs::write(snapshot.join("Qwen3-0.6B-Q4_0.gguf"), b"gguf").unwrap(); - let paths = AppPaths { - config_dir: dir.join("config"), - data_dir: dir.join("data"), - cache_dir, - }; - assert_eq!( - direct_llama_model_path(&paths, &runtime_dir, "Qwen3-0.6B-GGUF"), - Some(snapshot.join("Qwen3-0.6B-Q4_0.gguf")) - ); - fs::remove_dir_all(&dir).ok(); + fn rocm_channel_names_are_not_manifest_backends() { + // Installs record `rocm` or `vulkan`; `rocm-stable` and `rocm-nightly` are only + // directory names inside the `rocm` family, so a manifest naming one finds + // nothing and is refused as an unsupported backend, not as a missing install. + let dir = scratch_dir("channel-names"); + let mut manifest = test_manifest(dir.join("runtime")); + let stable = manifest.runtime_dir.join("bin/llamacpp/rocm-stable"); + fs::create_dir_all(&stable).unwrap(); + fs::write(stable.join(platform_binary_name("llama-server")), b"x").unwrap(); + assert!(direct_serve_binary(&manifest).is_some()); + for name in ["rocm-stable", "rocm-nightly"] { + manifest.backend_name = name.to_owned(); + assert_eq!(direct_serve_binary(&manifest), None, "{name}"); + let error = serving_route(&manifest, &serve_request("owner/repo:Q4_0"), true) + .unwrap_err() + .to_string(); + assert!( + error.contains("does not support GPU direct serving"), + "{error}" + ); + } + fs::remove_dir_all(dir).unwrap(); } fn test_manifest(runtime_dir: PathBuf) -> LemonadeInstallManifest { @@ -1473,4 +2632,421 @@ mod tests { "no Hugging Face cache directories could be determined" ); } + + /// A stand-in llama-server that stays up until it is signalled. + #[cfg(target_os = "linux")] + fn fake_llama_server(dir: &Path) -> PathBuf { + let server = dir.join("bin/llamacpp/vulkan/llama-server"); + fs::create_dir_all(server.parent().unwrap()).unwrap(); + // The script records its own PID before `exec`: afterwards the process + // is `sleep 30` and its command line no longer names the script, so the + // PID is the only reliable way to find it again. + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nprintf '%s' \"$$\" > \"$(dirname \"$0\")/server.pid\"\nexec sleep 30\n", + ) + .unwrap(); + server + } + + /// The PID the fake server recorded, if it got as far as running its script. + /// Waits briefly for it, so a server that was NOT stopped is seen. + #[cfg(target_os = "linux")] + fn recorded_server_pid(server: &Path) -> Option { + let pid_file = server.parent().unwrap().join("server.pid"); + for _ in 0..40 { + if let Some(pid) = fs::read_to_string(&pid_file) + .ok() + .and_then(|text| text.trim().parse().ok()) + { + return Some(pid); + } + std::thread::sleep(Duration::from_millis(50)); + } + None + } + + /// Kills and reaps a spawned child when dropped, so a failing assertion + /// cannot leave a fake server running. + #[cfg(target_os = "linux")] + struct KillOnDrop(Child); + + #[cfg(target_os = "linux")] + impl Drop for KillOnDrop { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + + #[cfg(target_os = "linux")] + fn process_exists(pid: u32) -> bool { + Path::new(&format!("/proc/{pid}")).exists() + } + + #[cfg(target_os = "linux")] + #[derive(Clone, Copy, Debug)] + enum DirectFail { + RecordRunning, + NotReady, + RecordReady, + Wait, + } + + #[cfg(target_os = "linux")] + struct FailingSteps(DirectFail); + + #[cfg(target_os = "linux")] + impl DirectServerSteps for FailingSteps { + fn record_running(&mut self, _server_pid: u32) -> Result<()> { + if matches!(self.0, DirectFail::RecordRunning) { + bail!("injected record_running failure"); + } + Ok(()) + } + fn wait_ready(&mut self) -> Result<()> { + if matches!(self.0, DirectFail::NotReady) { + bail!("injected readiness failure"); + } + Ok(()) + } + fn record_ready(&mut self, _server_pid: u32) -> Result<()> { + if matches!(self.0, DirectFail::RecordReady) { + bail!("injected record_ready failure"); + } + Ok(()) + } + fn announce_serving(&mut self) { + // Only a failure after the server became ready (the final wait) may + // follow an announcement; every earlier failure must not announce. + assert!( + matches!(self.0, DirectFail::Wait), + "{:?}: announced serving although a step before ready failed", + self.0 + ); + } + fn wait_exit(&mut self, _child: &mut Child) -> std::io::Result { + Err(std::io::Error::other("injected wait failure")) + } + } + + /// `wait_ready` gates `ready` on a chat completion as well as on + /// `/v1/models`: a server that lists the model but cannot answer a chat + /// request is not serving it. + #[test] + fn a_direct_server_that_lists_the_model_but_fails_its_chat_smoke_test_is_not_ready() { + use crate::test_http::{Reply, spawn_fixture}; + let (port, server) = spawn_fixture(Duration::from_secs(10), |recorded| { + if recorded.path.starts_with("/v1/models") { + Reply { + status: 200, + body: r#"{"data":[{"id":"owner/repo:Q4_0"}]}"#.to_owned(), + last: false, + } + } else { + Reply { + status: 500, + body: "{}".to_owned(), + last: true, + } + } + }); + let dir = scratch_dir("direct-smoke-fails"); + let request = ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: "owner/repo:Q4_0".to_owned(), + host: "127.0.0.1".to_owned(), + port, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path: dir.join("state.json"), + log_path: None, + engine_recipe: None, + }; + let runtime = LemonadeRuntime { + manifest: test_manifest(dir.join("runtime")), + }; + let model_path = dir.join("model.gguf"); + let mut steps = RealDirectServerSteps { + request: &request, + runtime: &runtime, + backend: "vulkan", + model_path: &model_path, + log_path: None, + ready_timeout: Duration::from_secs(5), + }; + let error = steps.wait_ready().unwrap_err().to_string(); + assert!(error.contains("chat-completion smoke test"), "{error}"); + let paths: Vec = server + .join() + .unwrap() + .into_iter() + .map(|recorded| recorded.path) + .collect(); + assert!( + paths + .iter() + .any(|path| path.starts_with("/v1/chat/completions")), + "{paths:?}" + ); + fs::remove_dir_all(dir).ok(); + } + + /// Every step succeeded, so how the server ends decides the record: a clean + /// exit is `stopped` and the serve returns `Ok`, a non-zero one is `failed` and + /// the serve returns the exit status. + #[cfg(target_os = "linux")] + #[test] + fn a_ready_direct_server_that_exits_is_recorded_by_how_it_exited() { + struct ReadySteps; + impl DirectServerSteps for ReadySteps { + fn record_running(&mut self, _server_pid: u32) -> Result<()> { + Ok(()) + } + fn wait_ready(&mut self) -> Result<()> { + Ok(()) + } + fn record_ready(&mut self, _server_pid: u32) -> Result<()> { + Ok(()) + } + fn announce_serving(&mut self) {} + } + for (code, status) in [(0, "stopped"), (1, "failed")] { + let dir = scratch_dir("direct-exit"); + let server = dir.join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + format!("#!/bin/sh\nexit {code}\n"), + ) + .unwrap(); + let state_path = dir.join("state.json"); + fs::write(&state_path, r#"{"status":"ready"}"#).unwrap(); + let mut child = ProcessCommand::new(&server).spawn().unwrap(); + let outcome = supervise_direct_server(&mut child, &state_path, &mut ReadySteps); + if code == 0 { + outcome.unwrap(); + } else { + let error = outcome.unwrap_err().to_string(); + assert!(error.contains("exited with status"), "{error}"); + } + let state: Value = + serde_json::from_str(&fs::read_to_string(&state_path).unwrap()).unwrap(); + assert_eq!(state["status"], status, "exit {code}: {state}"); + fs::remove_dir_all(dir).unwrap(); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn direct_server_step_failure_stops_the_child_and_marks_state_failed() { + for fail in [ + DirectFail::RecordRunning, + DirectFail::NotReady, + DirectFail::RecordReady, + DirectFail::Wait, + ] { + let dir = crate::test_support::scratch_dir("direct-supervise"); + let server = fake_llama_server(&dir); + let state_path = dir.join("state.json"); + fs::write(&state_path, r#"{"status":"running"}"#).unwrap(); + let mut guard = KillOnDrop(ProcessCommand::new(&server).spawn().unwrap()); + let pid = guard.0.id(); + assert!(process_exists(pid), "{fail:?}: fake server did not start"); + let error = supervise_direct_server(&mut guard.0, &state_path, &mut FailingSteps(fail)) + .unwrap_err() + .to_string(); + assert!( + error.contains("injected") || error.contains("waiting"), + "{error}" + ); + assert!( + !process_exists(pid), + "{fail:?}: server {pid} is still running after the failure" + ); + let state = fs::read_to_string(&state_path).unwrap(); + assert!( + state.contains(r#""status": "failed""#) || state.contains(r#""status":"failed""#), + "{fail:?}: {state}" + ); + fs::remove_dir_all(dir).unwrap(); + } + } + + /// A direct serve whose server never becomes ready must not leave a log line + /// claiming Lemonade is serving: the log is where a managed serve is read, and + /// the state next to it says `failed`. + #[cfg(target_os = "linux")] + #[test] + fn a_direct_serve_that_never_becomes_ready_does_not_log_that_it_is_serving() { + let dir = crate::test_support::scratch_dir("direct-not-ready"); + let server = fake_llama_server(&dir); + let model = dir.join("model.gguf"); + fs::write(&model, b"x").unwrap(); + let state_path = dir.join("state.json"); + fs::write(&state_path, r#"{"status":"starting"}"#).unwrap(); + let log_path = dir.join("serve.log"); + let request = ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: "owner/repo:Q4_0".to_owned(), + host: "127.0.0.1".to_owned(), + // Nothing listens here, so the readiness probe can only time out. + port: 11438, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path: state_path.clone(), + log_path: Some(log_path.clone()), + engine_recipe: None, + }; + let runtime = LemonadeRuntime { + manifest: test_manifest(dir.join("runtime")), + }; + let error = serve_direct_llama_server_within( + &request, + &runtime, + &LemonadeProcessEnvironment::default(), + &server, + &model, + Some(&log_path), + "test", + &ParentRuntimeEnvironment::rooted_at(&dir), + Duration::from_secs(1), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("did not become ready"), "{error}"); + let still_running = recorded_server_pid(&server) + .is_some_and(|pid| Path::new(&format!("/proc/{pid}")).exists()); + if still_running && let Some(pid) = recorded_server_pid(&server) { + let _ = terminate_pid(pid, true); + } + assert!(!still_running, "the server outlived the failed serve"); + let state = fs::read_to_string(&state_path).unwrap(); + assert!(state.contains("failed"), "{state}"); + let log = fs::read_to_string(&log_path).unwrap(); + assert!(log.contains("Launching Lemonade packaged vulkan"), "{log}"); + assert!(!log.contains("is serving on"), "{log}"); + fs::remove_dir_all(dir).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn direct_server_stops_when_state_cannot_be_written_after_spawn() { + let dir = crate::test_support::scratch_dir("direct-unwritable"); + let server = fake_llama_server(&dir); + let model = dir.join("model.gguf"); + fs::write(&model, b"x").unwrap(); + // A directory at the state path makes the first post-spawn write fail. + let state_path = dir.join("state.json"); + fs::create_dir_all(&state_path).unwrap(); + let request = ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: "owner/repo:Q4_0".to_owned(), + host: "127.0.0.1".to_owned(), + port: 11437, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path, + log_path: None, + engine_recipe: None, + }; + let runtime = LemonadeRuntime { + manifest: test_manifest(dir.join("runtime")), + }; + let result = serve_direct_llama_server( + &request, + &runtime, + &LemonadeProcessEnvironment::default(), + &server, + &model, + None, + "test", + &ParentRuntimeEnvironment::rooted_at(&dir), + ); + // A server that was not stopped runs its script and records its PID; one + // that was stopped first either never got that far or is gone by now. + let still_running = recorded_server_pid(&server) + .is_some_and(|pid| Path::new(&format!("/proc/{pid}")).exists()); + // Clean up before asserting so a failure cannot leave the server behind. + if still_running && let Some(pid) = recorded_server_pid(&server) { + let _ = terminate_pid(pid, true); + } + assert!(result.is_err()); + assert!(!still_running, "the direct server was left running"); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn serving_direct_command_preserves_alias_model_path_and_gpu_backend() { + let request = ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: "qwen-smoke".to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path: PathBuf::from("test-state"), + log_path: None, + engine_recipe: Some(EngineRecipeHint { + required_flags: vec!["--ctx-size".to_owned(), "8192".to_owned()], + ..EngineRecipeHint::default() + }), + }; + let server = Path::new("runtime/bin/llamacpp/vulkan/llama-build/llama-server"); + let model = Path::new("cache/model-Q4_0.gguf"); + let command = direct_llama_server_command(&request, server, model); + assert_eq!(command.get_program(), server.as_os_str()); + let args = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + args, + [ + "-m", + "cache/model-Q4_0.gguf", + "--host", + "127.0.0.1", + "--port", + "11435", + "--n-gpu-layers", + "999", + "--alias", + "qwen-smoke", + "--ctx-size", + "8192" + ] + ); + assert_eq!(llama_server_backend_label(server), "vulkan"); + assert!( + parse_models_ready(r#"{"data":[{"id":"qwen-smoke"}]}"#, &request.model_ref).unwrap() + ); + assert!( + !parse_models_ready(r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"#, &request.model_ref) + .unwrap() + ); + } + + #[test] + fn serving_regression_direct_readiness_requires_requested_alias() { + assert!(parse_models_ready(r#"{"data":[{"id":"qwen-smoke"}]}"#, "qwen-smoke").unwrap()); + assert!( + !parse_models_ready(r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"#, "qwen-smoke").unwrap() + ); + // The router deliberately normalizes known aliases, unlike direct serving. + assert!(models_payload_has_ready_model( + &json!({"data": [{ + "id": "Qwen3-0.6B-GGUF", "recipe_options": {"llamacpp_backend": "vulkan"} + }]}), + "qwen-smoke", + "vulkan" + )); + } } diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 609f9cad9b..78ea6c164c 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result, anyhow, bail}; +use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; use rocm_core::{AppPaths, DEFAULT_LOCAL_PORT, normalize_runtime_path_for_host, runtime_is_linux}; use rocm_engine_protocol::{ @@ -15,7 +15,7 @@ use rocm_engine_protocol::{ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::io::Read; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; mod archive_cache; @@ -32,9 +32,12 @@ mod footprint; mod install; mod llamacpp_backend; mod process; +mod router_serve; mod runtime_dir; mod state; #[cfg(test)] +mod test_http; +#[cfg(test)] mod test_support; // Re-exported so this stays reachable at its pre-split crate-root path — @@ -52,8 +55,6 @@ pub use footprint::{ pub(crate) const ENGINE_NAME: &str = "lemonade"; pub(crate) const DEFAULT_HOST: &str = "127.0.0.1"; pub(crate) const DEFAULT_MODEL: &str = "Qwen3-4B-Instruct-2507-GGUF"; -pub(crate) const DEFAULT_MODEL_REPO_DIR: &str = "models--unsloth--Qwen3-4B-Instruct-2507-GGUF"; -pub(crate) const DEFAULT_MODEL_GGUF: &str = "Qwen3-4B-Instruct-2507-Q4_K_M.gguf"; pub(crate) const LLAMACPP_RECIPE: &str = "llamacpp"; pub(crate) const ROCM_BACKEND_NAME: &str = "rocm"; #[cfg(feature = "e2e-test-hooks")] @@ -404,7 +405,7 @@ fn install_response(request: InstallRequest) -> Result { #[cfg(feature = "e2e-test-hooks")] if std::env::var_os(BACKEND_INSTALL_FAILURE_TEST_ENV).is_some() { crate::llamacpp_backend::install_llamacpp_backend_with_retry(|| { - bail!("scripted Lemonade backend install failure") + anyhow::bail!("scripted Lemonade backend install failure") })?; } let env_root = request @@ -473,7 +474,7 @@ fn install_response(request: InstallRequest) -> Result { fn resolve_model_response(request: ResolveModelRequest) -> Result { // Refuse an unservable model reference on the pre-launch round trip `rocm serve` - // makes, not only in `serve_hf_checkpoint`. That backstop runs three layers down, + // makes, not only in `require_checkpoint_variant`. That backstop runs three layers down, // inside the background engine process — which on Windows is spawned with no // stdout or stderr, and whose exit status nobody reads — so its refusal is // invisible there: the user gets an empty log, exit 0, and a service recorded as @@ -528,22 +529,7 @@ fn launch_service(mut request: LaunchRequest) -> Result { .engine_state_dir(ENGINE_NAME) .join(format!("{}.json", request.service_id)); let endpoint_url = crate::state::endpoint_url(&request.host, request.port); - let serve_request = ServeHttpRequest { - service_id: request.service_id.clone(), - model_ref: crate::state::resolve_lemonade_model_ref(&request.model_ref), - host: request.host.clone(), - port: request.port, - device_policy: request - .device_policy - .clone() - .unwrap_or(DevicePolicy::GpuRequired), - gpu_indices: rocm_engine_protocol::launch_gpu_indices(request.gpu_selection.as_ref()), - runtime_id: request.runtime_id.clone(), - env_id: request.env_id.clone(), - state_path: state_path.clone(), - log_path: Some(log_path.clone()), - engine_recipe: request.engine_recipe.clone(), - }; + let serve_request = launch_serve_request(&request, &state_path, &log_path); let current_exe = std::env::current_exe().context("failed to discover current Lemonade engine binary")?; let args = crate::process::serve_http_command_args(&serve_request); @@ -574,190 +560,182 @@ fn launch_service(mut request: LaunchRequest) -> Result { }) } -fn serve_http(mut request: ServeHttpRequest) -> Result<()> { - crate::device_policy::require_gpu_required(&request.device_policy)?; - // Verify a real GPU is available before launching anything: zero usable - // devices fails fast, `auto` resolves to the first present device (never an - // assumed device 0), and an explicit `--gpu` index is rejected when it is not - // actually present. This runs before any server process is spawned. - request.gpu_indices = crate::device_policy::resolve_serve_gpu_indices(&request.gpu_indices)?; - let mut runtime = crate::install::resolve_runtime()?; - let mut process_env = crate::process::lemonade_process_environment()?; - process_env.gpu_indices = request.gpu_indices.clone(); - let log_path = request.log_path.as_deref(); - crate::state::write_running_state(&request, &runtime, std::process::id(), None, "starting")?; - // A canonical Hugging Face checkpoint (owner/repo:variant) cannot be served under its - // exact name through Lemonade's model router (Lemonade renames registered models and - // its registry has several naming quirks). Download the GGUF and run a packaged - // llama-server on it directly with `--alias`, which serves it under exactly that name. - if let Some(checkpoint) = crate::direct_llama::parse_hf_checkpoint(&request.model_ref) { - return crate::direct_llama::serve_hf_checkpoint( - &request, - &runtime, - &process_env, - log_path, - &checkpoint, - ); +/// The serve request the wrapper process is started with. The model reference is +/// normalized once here, so a launch serves (and records) the same identity that +/// `resolve` reports for it: `qwen-smoke` launches as `Qwen3-0.6B-GGUF`. A +/// canonical Hugging Face checkpoint passes through unchanged. `serve-http` itself +/// never normalizes, so a direct serve-http run keeps exactly the name it was given. +fn launch_serve_request( + request: &LaunchRequest, + state_path: &Path, + log_path: &Path, +) -> ServeHttpRequest { + ServeHttpRequest { + service_id: request.service_id.clone(), + model_ref: crate::state::resolve_lemonade_model_ref(&request.model_ref), + host: request.host.clone(), + port: request.port, + device_policy: request + .device_policy + .clone() + .unwrap_or(DevicePolicy::GpuRequired), + gpu_indices: rocm_engine_protocol::launch_gpu_indices(request.gpu_selection.as_ref()), + runtime_id: request.runtime_id.clone(), + env_id: request.env_id.clone(), + state_path: state_path.to_path_buf(), + log_path: Some(log_path.to_path_buf()), + engine_recipe: request.engine_recipe.clone(), } - if runtime_is_linux() - && let Some(server) = crate::direct_llama::find_llama_server_binary(&runtime.manifest) - { - crate::direct_llama::ensure_direct_llama_model_available( - &request, - &runtime, - &process_env, - log_path, - )?; - let backend = crate::direct_llama::llama_server_backend_label(&server); - return crate::direct_llama::serve_direct_llama_server( - &request, - &runtime, - &process_env, - &server, - log_path, - &anyhow!("using Lemonade packaged {backend} llama-server directly on Linux"), - ); +} + +/// Everything a serve needs from its environment, resolved once up front so tests +/// can supply their own (a confined cache and parent environment, a fake runtime). +struct ServeContext { + runtime: crate::install::LemonadeRuntime, + process_env: crate::process::LemonadeProcessEnvironment, + paths: AppPaths, + parent: crate::runtime_dir::ParentRuntimeEnvironment, + /// Whether this host is Linux: the route choice and the router-to-direct + /// fallback both depend on it, and a test cannot change the host. + is_linux: bool, +} + +fn serve_http(request: ServeHttpRequest) -> Result<()> { + serve_http_resolving(request, |request| { + crate::device_policy::require_gpu_required(&request.device_policy)?; + // Verify a real GPU is available before launching anything: zero usable + // devices fails fast, `auto` resolves to the first present device (never an + // assumed device 0), and an explicit `--gpu` index is rejected when it is not + // actually present. This runs before any server process is spawned. + request.gpu_indices = + crate::device_policy::resolve_serve_gpu_indices(&request.gpu_indices)?; + let runtime = crate::install::resolve_runtime()?; + let mut process_env = crate::process::lemonade_process_environment()?; + process_env.gpu_indices = request.gpu_indices.clone(); + Ok(ServeContext { + runtime, + process_env, + paths: AppPaths::discover()?, + parent: crate::runtime_dir::ParentRuntimeEnvironment::current(), + is_linux: runtime_is_linux(), + }) + }) +} + +/// Run a serve and make its outcome visible in the persisted state: any error, +/// whichever step raised it, leaves a state file that exists at `failed` rather +/// than at the `starting`/`running` it last recorded, and the error is returned +/// unchanged. Servers spawned inside [`serve_with_context`] are stopped by the +/// code that spawned them, which owns the child. +fn serve_http_resolving( + mut request: ServeHttpRequest, + resolve: impl FnOnce(&mut ServeHttpRequest) -> Result, +) -> Result<()> { + let state_path = request.state_path.clone(); + let result = resolve(&mut request).and_then(|context| serve_with_context(&request, context)); + if result.is_err() && state_path.is_file() { + let _ = crate::state::mark_json_status(&state_path, "failed"); } - let mut child = crate::process::spawn_lemond( - &runtime.manifest, - &request.host, - request.port, - log_path, - &process_env, - )?; - crate::state::write_running_state( - &request, - &runtime, - std::process::id(), - Some(child.id()), - "running", - )?; - crate::process::wait_for_lemonade_cli_status( - &runtime.manifest, - &request.host, - request.port, - Duration::from_secs(30), - log_path, - &process_env, - ) - .context("Lemonade server did not become ready")?; - let backend = crate::llamacpp_backend::ensure_best_llamacpp_backend( - &mut runtime.manifest, - &request.host, - request.port, - &process_env, - false, - ) - .context("failed to select a supported Lemonade llama.cpp backend")?; - let load_result = crate::process::run_lemonade_model_load( - &runtime.manifest, - &request.host, - request.port, - &request.model_ref, - &backend, - request.engine_recipe.as_ref(), - log_path, - &process_env, - ); - let router_ready = load_result.is_ok() - && crate::process::query_loaded_model_endpoint( - &crate::state::endpoint_url(&request.host, request.port), - &request.model_ref, - &backend, - ) - .unwrap_or(false) - && crate::process::query_chat_smoke_endpoint( - &request.host, - request.port, - &request.model_ref, - ) - .unwrap_or(false); - if let Err(error) = load_result { - if runtime_is_linux() - && let Some(direct_server) = - crate::direct_llama::find_llama_server_binary(&runtime.manifest) - { - let _ = crate::state::terminate_pid(child.id(), true); - let _ = child.wait(); - return crate::direct_llama::serve_direct_llama_server( - &request, + result +} + +fn serve_with_context(request: &ServeHttpRequest, context: ServeContext) -> Result<()> { + let ServeContext { + runtime, + process_env, + paths, + parent, + is_linux, + } = context; + let log_path = request.log_path.as_deref(); + crate::state::write_running_state(request, &runtime, std::process::id(), None, "starting")?; + let route = crate::direct_llama::serving_route(&runtime.manifest, request, is_linux)?; + let router_model_ref = match route { + crate::direct_llama::ServeRoute::Direct { model, server } => { + let model_path = crate::direct_llama::ensure_model_in( + &paths, + request, &runtime, &process_env, - &direct_server, log_path, - &error, - ); - } - return Err(error).with_context(|| { - format!( - "failed to load {} with Lemonade {LLAMACPP_RECIPE}:{backend}", - request.model_ref - ) - }); - } - if !router_ready { - let error = anyhow!( - "Lemonade load completed but the endpoint did not report a {LLAMACPP_RECIPE}:{backend}-loaded model" - ); - if runtime_is_linux() - && let Some(direct_server) = - crate::direct_llama::find_llama_server_binary(&runtime.manifest) - { - let _ = crate::state::terminate_pid(child.id(), true); - let _ = child.wait(); + &model, + &parent, + )?; + let backend = crate::direct_llama::llama_server_backend_label(&server); return crate::direct_llama::serve_direct_llama_server( - &request, + request, &runtime, &process_env, - &direct_server, + &server, + &model_path, log_path, - &error, + &format!("using Lemonade packaged {backend} llama-server directly"), + &parent, ); } - return Err(error).with_context(|| { - format!( - "failed to verify {} with Lemonade {LLAMACPP_RECIPE}:{backend}", - request.model_ref + crate::direct_llama::ServeRoute::Router { model_ref } => model_ref, + }; + let mut steps = crate::router_serve::LemondRouterSteps { + runtime, + request, + process_env: &process_env, + log_path, + parent, + child: None, + }; + let result = crate::router_serve::serve_router(request, &mut steps, &router_model_ref); + with_direct_fallback( + result, + &paths, + request, + &steps.runtime, + &process_env, + log_path, + &steps.parent, + is_linux, + ) +} + +/// Route a router serve's outcome. The router already stopped its server and, +/// for a load or verification failure, left the state at `running`. Only a model load or verification failure may be +/// retried, and only through the GPU llama-server of the backend Lemonade +/// selected; every other outcome is returned as it is. +#[allow(clippy::too_many_arguments)] +fn with_direct_fallback( + result: Result<()>, + paths: &AppPaths, + request: &ServeHttpRequest, + runtime: &crate::install::LemonadeRuntime, + process_env: &crate::process::LemonadeProcessEnvironment, + log_path: Option<&Path>, + parent: &crate::runtime_dir::ParentRuntimeEnvironment, + is_linux: bool, +) -> Result<()> { + match result { + Err(error) if crate::router_serve::is_router_load_failure(&error) => { + crate::direct_llama::serve_direct_after_router_failure( + paths, + request, + runtime, + process_env, + log_path, + parent, + is_linux, + error, ) - }); - } - crate::state::merge_json_state( - &request.state_path, - &json!({ - "status": "ready", - "server_pid": child.id(), - // Identity token for the server PID, captured while the child is alive. - "server_start_ticks": rocm_core::process_start_ticks(child.id()), - "backend_state": "ready", - "backend_requested": backend, - "load_response": { - "status": "loaded", - "method": "lemonade-cli", - "model_name": request.model_ref, - "llamacpp_backend": backend - }, - }), - )?; - let status = child.wait().context("failed waiting for Lemonade server")?; - crate::state::mark_json_status( - &request.state_path, - if status.success() { - "stopped" - } else { - "failed" - }, - )?; - if status.success() { - Ok(()) - } else { - bail!("Lemonade server exited with status {status}") + } + result => result, } } fn healthcheck_service(request: HealthcheckRequest) -> Result { rocm_core::require_nonempty(&request.service_id, "service_id")?; let files = crate::state::service_files(&request.service_id)?; - let state = crate::state::read_service_state(&files.state_path).ok(); + healthcheck_service_at(&files.state_path) +} + +fn healthcheck_service_at(state_path: &Path) -> Result { + let state = crate::state::read_service_state(state_path).ok(); let endpoint_url = state .as_ref() .and_then(crate::state::endpoint_url_from_state); @@ -791,12 +769,7 @@ fn healthcheck_service(request: HealthcheckRequest) -> Result(value: &T) -> Result<()> { mod tests { use super::*; + /// Characterization: health and chat read the served name from the persisted + /// state. This holds with or without this change (`healthcheck_service_at` + /// already preferred `canonical_model_id`), so it guards the reading, not the + /// writing; `router_state_keeps_requested_text_and_records_router_name` covers + /// what a router serve persists. + #[test] + fn health_and_chat_read_the_served_name_from_the_persisted_state() { + use crate::test_http::{Reply, spawn_fixture}; + for (tag, served_name) in [("router", "Qwen3-0.6B-GGUF"), ("direct", "qwen-smoke")] { + let expected = served_name.to_owned(); + let (port, server) = spawn_fixture(Duration::from_secs(5), move |recorded| { + let chat = recorded.method == "POST"; + Reply { + status: 200, + body: if chat { + json!({"choices": [{"message": {"content": "ok"}}]}) + } else { + json!({"all_models_loaded": [{"model_name": expected, "recipe_options": {"llamacpp_backend": "vulkan"}}]}) + } + .to_string(), + last: chat, + } + }); + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let request = ServeHttpRequest { + service_id: tag.to_owned(), + model_ref: "qwen-smoke".to_owned(), + host: "127.0.0.1".to_owned(), + port, + device_policy: DevicePolicy::GpuRequired, + gpu_indices: vec![1], + runtime_id: None, + env_id: None, + state_path: dir.join("state.json"), + log_path: None, + engine_recipe: None, + }; + let runtime = crate::install::LemonadeRuntime { + manifest: crate::install::LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: dir.join("lemonade"), + backend_recipe: LLAMACPP_RECIPE.to_owned(), + backend_name: "vulkan".to_owned(), + installed_at_unix_ms: 0, + }, + }; + if tag == "direct" { + crate::state::write_running_state( + &request, + &runtime, + std::process::id(), + None, + "ready", + ) + .unwrap(); + } else { + crate::state::write_running_state_for_model( + &request, + &runtime, + std::process::id(), + None, + "ready", + served_name, + ) + .unwrap(); + } + crate::state::merge_json_state( + &request.state_path, + &json!({"backend_requested": "vulkan"}), + ) + .unwrap(); + let response = healthcheck_service_at(&request.state_path).unwrap(); + let seen = server.join().unwrap(); + let chat = seen + .iter() + .find(|recorded| recorded.method == "POST") + .expect("health inference request was not received"); + assert_eq!(chat.path, "/v1/chat/completions"); + let chat_model = serde_json::from_str::(&chat.body).unwrap()["model"] + .as_str() + .unwrap() + .to_owned(); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(response.status, "ready"); + assert_eq!(state["model_ref"], "qwen-smoke"); + assert_eq!(chat_model, served_name); + assert_eq!(state["canonical_model_id"], served_name); + assert!(state.get(rocm_core::INFERENCE_VERIFIED_STATE_KEY).is_some()); + } + } + + fn launch_request(model_ref: &str) -> LaunchRequest { + LaunchRequest { + service_id: "svc".to_owned(), + env_id: None, + runtime_id: None, + model_ref: model_ref.to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: None, + endpoint_mode: None, + engine_recipe: None, + gpu_selection: None, + } + } + + #[test] + fn launch_serves_the_identity_resolve_reports() { + // The engine's Launch method normalizes the requested reference once, so what + // it serves and records is what `resolve` reports for it. `serve-http` + // itself never normalizes (direct serving keeps the exact name it is given). + for (requested, reported) in [ + ("qwen-smoke", "Qwen3-0.6B-GGUF"), + ("assistant", DEFAULT_MODEL), + ("owner/repo:Q4_0", "owner/repo:Q4_0"), + ("Some-Registered-Model", "Some-Registered-Model"), + ] { + let resolved = resolve_model_response(ResolveModelRequest { + model_ref: requested.to_owned(), + runtime_id: None, + device_policy: None, + recipe_override: None, + engine_recipe: None, + }) + .unwrap(); + assert_eq!(resolved.canonical_model_id, reported); + let serve = launch_serve_request( + &launch_request(requested), + Path::new("state.json"), + Path::new("service.log"), + ); + assert_eq!(serve.model_ref, resolved.canonical_model_id, "{requested}"); + // The wrapper process is started with that same identity. + let args = crate::process::serve_http_command_args(&serve); + assert_eq!(args[2], reported, "{args:?}"); + } + } + + /// A confined serve environment: a fake runtime under `dir`, a model cache under + /// `dir/cache`, and a parent environment rooted at `dir`. + #[cfg(target_os = "linux")] + fn fake_context(dir: &Path, manifest: install::LemonadeInstallManifest) -> ServeContext { + ServeContext { + runtime: crate::install::LemonadeRuntime { manifest }, + process_env: crate::process::LemonadeProcessEnvironment::default(), + paths: AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }, + parent: crate::runtime_dir::ParentRuntimeEnvironment::rooted_at(dir), + is_linux: true, + } + } + + #[cfg(target_os = "linux")] + fn fake_request(dir: &Path, model_ref: &str) -> ServeHttpRequest { + ServeHttpRequest { + service_id: "svc".to_owned(), + model_ref: model_ref.to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path: dir.join("state.json"), + log_path: None, + engine_recipe: None, + } + } + + #[cfg(target_os = "linux")] + fn install_fake_llama_server(manifest: &install::LemonadeInstallManifest, script: &str) { + crate::test_http::fake_runtime::write_executable( + &manifest + .runtime_dir + .join("bin/llamacpp/vulkan/llama-server"), + script, + ); + } + + #[cfg(target_os = "linux")] + fn write_registry(manifest: &install::LemonadeInstallManifest) { + let resources = manifest.runtime_dir.join("resources"); + std::fs::create_dir_all(&resources).unwrap(); + std::fs::write( + resources.join("server_models.json"), + json!({ + "Qwen3-0.6B-GGUF": { + "recipe": "llamacpp", + "checkpoint": "owner/serve-test-repo:Q4_0", + "suggested": true, + "size": 0.38, + "labels": ["reasoning"] + } + }) + .to_string(), + ) + .unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn serve_errors_before_the_server_starts_leave_state_failed() { + use crate::test_http::fake_runtime; + // Each case fails somewhere between the `starting` state being written and a + // server being spawned. None may leave the state at `starting`, and the + // error that reaches the caller is the original one. + for (case, model_ref, with_binary, pull_exit, expected) in [ + // serving_route: the selected backend's binary is missing. + ( + "missing binary", + "owner/serve-test-repo:Q4_0", + false, + 0, + "no selected GPU llama-server", + ), + // serving_route: a checkpoint without a variant. + ( + "invalid checkpoint", + "owner/serve-test-repo", + true, + 0, + "needs an explicit GGUF variant", + ), + // ensure_model_in: the download fails. + ( + "failed pull", + "owner/serve-test-repo:Q4_0", + true, + 7, + "Lemonade pull of `owner/serve-test-repo:Q4_0` failed", + ), + // ensure_model_in: the download succeeds but yields no usable GGUF. + ( + "pull without file", + "owner/serve-test-repo:Q4_0", + true, + 0, + "did not download", + ), + ] { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = fake_runtime::manifest(dir, pull_exit, None); + if with_binary { + install_fake_llama_server(&manifest, "#!/bin/sh\nexit 0\n"); + } + let request = fake_request(dir, model_ref); + let context = fake_context(dir, manifest); + let error = serve_http_resolving(request.clone(), |_| Ok(context)) + .expect_err(case) + .to_string(); + assert!(error.contains(expected), "{case}: {error}"); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed", "{case}"); + assert_eq!(state["model_ref"], model_ref, "{case}"); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn serve_error_while_resolving_the_runtime_marks_launch_state_failed() { + // `launch` wrote `starting` before the wrapper ran; a wrapper that fails to + // resolve its runtime must not leave that behind. + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let request = fake_request(dir, "qwen-smoke"); + std::fs::write( + &request.state_path, + json!({"status": "starting"}).to_string(), + ) + .unwrap(); + let error = serve_http_resolving(request.clone(), |_| anyhow::bail!("no runtime")) + .unwrap_err() + .to_string(); + assert_eq!(error, "no runtime"); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed"); + // With no state file there is nothing to mark, and none is invented. + let missing = fake_request(&dir.join("elsewhere"), "qwen-smoke"); + let _ = serve_http_resolving(missing.clone(), |_| anyhow::bail!("no runtime")); + assert!(!missing.state_path.exists()); + } + + #[cfg(target_os = "linux")] + #[test] + fn direct_serve_through_serve_http_uses_the_requested_alias() { + use crate::test_http::{Reply, fake_runtime, spawn_fixture}; + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = fake_runtime::manifest(dir, 0, None); + write_registry(&manifest); + let args_file = dir.join("server-args.txt"); + install_fake_llama_server( + &manifest, + &format!( + "#!/bin/sh\nprintf '%s\\n' \"$@\" > '{}'\nexit 0\n", + args_file.display() + ), + ); + // The registered checkpoint is already cached, so nothing is downloaded. + let gguf = dir.join( + "cache/huggingface/hub/models--owner--serve-test-repo/snapshots/rev/model-Q4_0.gguf", + ); + std::fs::create_dir_all(gguf.parent().unwrap()).unwrap(); + std::fs::write(&gguf, b"gguf").unwrap(); + let (port, server) = spawn_fixture(Duration::from_secs(10), |recorded| { + let chat = recorded.method == "POST"; + Reply { + status: 200, + body: if chat { + json!({"choices": [{"message": {"content": "ok"}}]}) + } else { + json!({"data": [{"id": "qwen-smoke"}]}) + } + .to_string(), + last: chat, + } + }); + let mut request = fake_request(dir, "qwen-smoke"); + request.port = port; + let context = fake_context(dir, manifest); + serve_http_resolving(request.clone(), |_| Ok(context)).unwrap(); + + let args = std::fs::read_to_string(&args_file) + .unwrap() + .lines() + .map(str::to_owned) + .collect::>(); + let after = |flag: &str| { + let at = args.iter().position(|arg| arg == flag).expect(flag); + args[at + 1].clone() + }; + // Served under exactly the name that was requested, not the registered name. + assert_eq!(after("--alias"), "qwen-smoke", "{args:?}"); + assert_eq!(after("-m"), gguf.display().to_string(), "{args:?}"); + assert_eq!(after("--port"), port.to_string(), "{args:?}"); + let chat = server + .join() + .unwrap() + .into_iter() + .find(|recorded| recorded.method == "POST") + .expect("smoke request"); + assert_eq!( + serde_json::from_str::(&chat.body).unwrap()["model"], + "qwen-smoke" + ); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["model_ref"], "qwen-smoke"); + assert_eq!(state["canonical_model_id"], "qwen-smoke"); + assert_eq!(state["status"], "stopped"); + } + + #[cfg(target_os = "linux")] + fn fallback_fixture( + dir: &Path, + script: &str, + ) -> ( + ServeContext, + ServeHttpRequest, + std::path::PathBuf, + u16, + std::thread::JoinHandle>, + ) { + fallback_fixture_with(dir, script, true) + } + + /// `cached` is whether the GGUF is already in the hub cache. When it is not, + /// the fake `lemonade pull` puts it there, so the fallback runs a pull. + #[cfg(target_os = "linux")] + fn fallback_fixture_with( + dir: &Path, + script: &str, + cached: bool, + ) -> ( + ServeContext, + ServeHttpRequest, + std::path::PathBuf, + u16, + std::thread::JoinHandle>, + ) { + use crate::test_http::{Reply, fake_runtime, spawn_fixture}; + let pulled = "models--owner--serve-test-repo/snapshots/rev/model-Q4_0.gguf"; + let manifest = fake_runtime::manifest(dir, 0, (!cached).then_some(pulled)); + write_registry(&manifest); + install_fake_llama_server(&manifest, script); + let gguf = dir.join("cache/huggingface/hub").join(pulled); + if cached { + std::fs::create_dir_all(gguf.parent().unwrap()).unwrap(); + std::fs::write(&gguf, b"gguf").unwrap(); + } + let (port, server) = spawn_fixture(Duration::from_secs(10), |recorded| { + let chat = recorded.method == "POST"; + Reply { + status: 200, + body: if chat { + json!({"choices": [{"message": {"content": "ok"}}]}) + } else { + json!({"data": [{"id": "qwen-smoke"}]}) + } + .to_string(), + last: chat, + } + }); + let mut request = fake_request(dir, "qwen-smoke"); + request.port = port; + request.log_path = Some(dir.join("serve.log")); + (fake_context(dir, manifest), request, gguf, port, server) + } + + #[cfg(target_os = "linux")] + fn args_script(args_file: &Path) -> String { + format!( + "#!/bin/sh\nprintf '%s\\n' \"$@\" > '{}'\nexit 0\n", + args_file.display() + ) + } + + #[cfg(target_os = "linux")] + fn marked_load_failure() -> anyhow::Error { + anyhow::Error::msg("load exploded").context(crate::router_serve::RouterLoadFailure( + "failed to load qwen-smoke".to_owned(), + )) + } + + #[cfg(target_os = "linux")] + #[test] + fn router_load_failure_falls_back_to_the_selected_gpu_llama_server() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let args_file = dir.join("server-args.txt"); + let (context, request, gguf, port, server) = + fallback_fixture(dir, &args_script(&args_file)); + with_direct_fallback( + Err(marked_load_failure()), + &context.paths, + &request, + &context.runtime, + &context.process_env, + request.log_path.as_deref(), + &context.parent, + true, + ) + .unwrap(); + server.join().unwrap(); + + // The state shows a direct serve that ran, under the requested name. + let args = std::fs::read_to_string(&args_file).unwrap(); + assert!( + args.contains(&format!("-m\n{}\n", gguf.display())), + "{args}" + ); + assert!(args.contains(&format!("--port\n{port}\n")), "{args}"); + assert!(args.contains("--alias\nqwen-smoke\n"), "{args}"); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "stopped"); + // The message names the backend of the server it launched, derived from + // the same path, and says it is not ROCm. + let log = std::fs::read_to_string(request.log_path.as_ref().unwrap()).unwrap(); + assert!( + log.contains("serving it directly with the selected vulkan GPU llama-server"), + "{log}" + ); + assert!(log.contains("load exploded"), "{log}"); + assert!( + log.contains("Lemonade is serving on the vulkan GPU backend, not ROCm"), + "{log}" + ); + assert!(!log.contains("cpu"), "{log}"); + } + + /// The router leaves the state at `running`, naming its own server. While the + /// fallback pulls the model, a reader must see `starting` and no server pid: + /// `identity_from_state` prefers `server_pid`, so a dead router's pid left in + /// place would point `stop` at the wrong process. The fake `lemonade pull` + /// copies the state file as it is at that moment, and the server then runs to + /// completion. + #[cfg(target_os = "linux")] + #[test] + fn the_state_says_starting_while_the_fallback_pulls_the_model() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let args_file = dir.join("server-args.txt"); + let (context, request, _gguf, _port, server) = + fallback_fixture_with(dir, &args_script(&args_file), false); + // What the failed router serve left behind. + crate::state::merge_json_state( + &request.state_path, + &serde_json::json!({ + "status": "running", + "pid": std::process::id(), + "server_pid": 4_000_000, + "server_start_ticks": 7, + }), + ) + .unwrap(); + with_direct_fallback( + Err(marked_load_failure()), + &context.paths, + &request, + &context.runtime, + &context.process_env, + request.log_path.as_deref(), + &context.parent, + true, + ) + .unwrap(); + server.join().unwrap(); + let during: serde_json::Value = serde_json::from_slice( + &std::fs::read(dir.join("state-at-pull.json")).expect("a pull ran"), + ) + .unwrap(); + assert_eq!(during["status"], "starting", "{during}"); + assert!(during["server_pid"].is_null(), "{during}"); + assert!(during["server_start_ticks"].is_null(), "{during}"); + // Until the direct server runs, the launcher is the process to stop. + assert_eq!( + crate::state::identity_from_state(&during).map(|identity| identity.pid), + Some(std::process::id()), + "{during}" + ); + // The fallback then went on to serve. + assert!(args_file.exists(), "the direct server ran"); + } + + /// The router leaves the state at `running`; the fallback sets `starting` + /// before it downloads or spawns and puts `failed` back when it does not + /// produce a server, so a reader never sees a stuck `starting` after a + /// fallback that does not. + #[cfg(target_os = "linux")] + #[test] + fn a_fallback_that_cannot_get_the_model_ends_failed_not_starting() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + // `lemonade pull` exits 1 and no GGUF is cached. + let manifest = crate::test_http::fake_runtime::manifest(dir, 1, None); + write_registry(&manifest); + install_fake_llama_server(&manifest, "#!/bin/sh\nexit 0\n"); + let mut request = fake_request(dir, "qwen-smoke"); + request.log_path = Some(dir.join("serve.log")); + let context = fake_context(dir, manifest); + let error = with_direct_fallback( + Err(marked_load_failure()), + &context.paths, + &request, + &context.runtime, + &context.process_env, + request.log_path.as_deref(), + &context.parent, + true, + ) + .unwrap_err(); + let text = format!("{error:#}"); + assert!(text.contains("load exploded"), "{text}"); + assert!(text.contains("fallback also failed"), "{text}"); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed", "{state}"); + } + + /// The Windows half of the fallback rule, through the whole serve: a router + /// that fails to verify its model is not retried directly when the host is not + /// Linux, though a GPU llama-server is installed and the model is cached. + /// `is_linux` comes from the serve context, so this fails if either the route + /// choice or the fallback reads the host instead. + #[cfg(target_os = "linux")] + #[test] + fn a_router_load_failure_is_not_served_directly_when_the_host_is_not_linux() { + use crate::test_http::{Reply, fake_runtime, spawn_fixture}; + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = fake_runtime::manifest(dir, 0, None); + // `lemonade backends` lists vulkan as installed; every other call succeeds. + fake_runtime::write_executable( + &manifest.lemonade, + "#!/bin/sh\nif [ \"$1\" = backends ]; then\n echo 'llamacpp vulkan installed ok'\nfi\nexit 0\n", + ); + write_registry(&manifest); + let args_file = dir.join("server-args.txt"); + install_fake_llama_server(&manifest, &args_script(&args_file)); + let gguf = dir.join( + "cache/huggingface/hub/models--owner--serve-test-repo/snapshots/rev/model-Q4_0.gguf", + ); + std::fs::create_dir_all(gguf.parent().unwrap()).unwrap(); + std::fs::write(&gguf, b"gguf").unwrap(); + // The endpoint answers, but never lists the model: the router's load is + // verified false, which is a tagged load failure. + let (port, _server) = spawn_fixture(Duration::from_secs(3), |_| Reply { + status: 200, + body: json!({"data": []}).to_string(), + last: false, + }); + let mut request = fake_request(dir, "qwen-smoke"); + request.port = port; + request.log_path = Some(dir.join("serve.log")); + let mut context = fake_context(dir, manifest); + context.is_linux = false; + let error = serve_http_resolving(request.clone(), |_| Ok(context)) + .unwrap_err() + .to_string(); + assert!(error.contains("failed to verify qwen-smoke"), "{error}"); + assert!(!args_file.exists(), "no llama-server may have started"); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed", "{state}"); + } + + #[cfg(target_os = "linux")] + #[test] + fn only_a_router_load_failure_may_fall_back_and_only_on_linux() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let args_file = dir.join("server-args.txt"); + let (context, request, _gguf, _port, server) = + fallback_fixture(dir, &args_script(&args_file)); + let run = |result: Result<()>, is_linux: bool| { + with_direct_fallback( + result, + &context.paths, + &request, + &context.runtime, + &context.process_env, + request.log_path.as_deref(), + &context.parent, + is_linux, + ) + }; + // A server that never became ready, a failed backend selection or a later + // exit is returned as it is, though a direct route is open. + let other = run( + Err(anyhow::anyhow!("Lemonade server did not become ready")), + true, + ) + .unwrap_err() + .to_string(); + assert_eq!(other, "Lemonade server did not become ready"); + // Off Linux the router's own error is returned unchanged. + let windows = run(Err(marked_load_failure()), false) + .unwrap_err() + .to_string(); + assert_eq!(windows, "failed to load qwen-smoke"); + assert!(!args_file.exists(), "no llama-server may have started"); + assert!(!request.log_path.as_ref().unwrap().exists()); + // The state the router left is settled by the fallback; with no direct + // route open the failure is final. + assert_eq!( + crate::state::read_service_state(&request.state_path).unwrap()["status"], + "failed" + ); + drop(server); + } + + #[cfg(target_os = "linux")] + #[test] + fn fallback_never_serves_through_a_missing_or_cpu_backend() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = crate::test_http::fake_runtime::manifest(dir, 0, None); + write_registry(&manifest); + let request = fake_request(dir, "qwen-smoke"); + // The selected GPU backend has no llama-server: nothing to fall back to. + assert!(crate::direct_llama::direct_fallback_route(&manifest, &request, true).is_none()); + // Only a CPU llama-server exists and the backend is CPU: still refused. + let mut cpu = manifest.clone(); + cpu.backend_name = "cpu".to_owned(); + crate::test_http::fake_runtime::write_executable( + &cpu.runtime_dir.join("bin/llamacpp/cpu/llama-server"), + "#!/bin/sh\nexit 0\n", + ); + assert!(crate::direct_llama::direct_fallback_route(&cpu, &request, true).is_none()); + // With the selected GPU backend installed the route opens on Linux only. + install_fake_llama_server(&manifest, "#!/bin/sh\nexit 0\n"); + assert!(crate::direct_llama::direct_fallback_route(&manifest, &request, true).is_some()); + assert!(crate::direct_llama::direct_fallback_route(&manifest, &request, false).is_none()); + } + #[test] fn stop_scope_targets_only_the_recorded_lemonade_server() { assert_eq!(STOP_SCOPE, rocm_core::KillScope::Single); diff --git a/engines/lemonade/src/llamacpp_backend.rs b/engines/lemonade/src/llamacpp_backend.rs index 5674afe21b..48a9caf91b 100644 --- a/engines/lemonade/src/llamacpp_backend.rs +++ b/engines/lemonade/src/llamacpp_backend.rs @@ -3,17 +3,18 @@ // SPDX-License-Identifier: MIT use anyhow::{Context, Result, bail}; -use std::path::PathBuf; +use std::fs; +use std::path::{Path, PathBuf}; use std::process::{Command as ProcessCommand, Stdio}; use std::time::Duration; -use crate::direct_llama::find_binary_in; use crate::install::{LemonadeInstallManifest, platform_binary_name}; use crate::process::{ - LemonadeProcessEnvironment, apply_lemonade_process_environment, hide_child_console_window, - lemonade_process_environment, run_lemonade_backend_install, spawn_lemond, - wait_for_lemonade_cli_status, + LemonadeProcessEnvironment, apply_lemonade_process_environment_with_parent, + hide_child_console_window, lemonade_process_environment, run_lemonade_backend_install, + spawn_lemond, wait_for_lemonade_cli_status, }; +use crate::runtime_dir::ParentRuntimeEnvironment; use crate::state::{free_local_port, terminate_pid}; use crate::{DEFAULT_HOST, LLAMACPP_RECIPE, ROCM_BACKEND_NAME}; @@ -73,7 +74,11 @@ pub(crate) fn best_llamacpp_backend_for_host( log_path, &process_env, )?; - let listing = run_lemonade_backends_list(manifest, &process_env)?; + let listing = run_lemonade_backends_list( + manifest, + &process_env, + &ParentRuntimeEnvironment::current(), + )?; let backends = parse_llamacpp_backend_statuses(&listing); Ok(select_best_llamacpp_backend(&backends).map(|(name, _)| name)) })(); @@ -112,7 +117,28 @@ pub(crate) fn ensure_best_llamacpp_backend( process_env: &LemonadeProcessEnvironment, force_reinstall: bool, ) -> Result { - let listing = run_lemonade_backends_list(manifest, process_env)?; + ensure_best_llamacpp_backend_with_parent( + manifest, + host, + port, + process_env, + force_reinstall, + &ParentRuntimeEnvironment::current(), + ) +} + +/// [`ensure_best_llamacpp_backend`] with the parent runtime environment supplied +/// by the caller, so a serve that was given a confined parent never reads or +/// creates anything in the host's own. +pub(crate) fn ensure_best_llamacpp_backend_with_parent( + manifest: &mut LemonadeInstallManifest, + host: &str, + port: u16, + process_env: &LemonadeProcessEnvironment, + force_reinstall: bool, + parent: &ParentRuntimeEnvironment, +) -> Result { + let listing = run_lemonade_backends_list(manifest, process_env, parent)?; let backends = parse_llamacpp_backend_statuses(&listing); let Some((backend, already_installed)) = select_best_llamacpp_backend(&backends) else { bail!( @@ -132,7 +158,7 @@ pub(crate) fn ensure_best_llamacpp_backend( } else { eprintln!("Installing Lemonade {LLAMACPP_RECIPE}:{backend} backend..."); install_llamacpp_backend_with_retry(|| { - run_lemonade_backend_install(manifest, host, port, &backend, process_env) + run_lemonade_backend_install(manifest, host, port, &backend, process_env, parent) })?; } Ok(backend) @@ -214,6 +240,7 @@ fn describe_llamacpp_backends(backends: &[(String, String)]) -> String { fn run_lemonade_backends_list( manifest: &LemonadeInstallManifest, process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, ) -> Result { let mut command = ProcessCommand::new(&manifest.lemonade); command @@ -221,7 +248,7 @@ fn run_lemonade_backends_list( .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - apply_lemonade_process_environment(&mut command, process_env)?; + apply_lemonade_process_environment_with_parent(&mut command, process_env, parent)?; hide_child_console_window(&mut command); let output = command .output() @@ -241,30 +268,54 @@ fn run_lemonade_backends_list( /// replacing one in place, so all three names can accumulate across attempts. pub(crate) const ROCM_LLAMACPP_BACKEND_DIRS: [&str; 3] = ["rocm-stable", "rocm-nightly", "rocm"]; -/// Like [`crate::direct_llama::find_llama_server_binary`], but scoped to the single backend Lemonade -/// actually attempted (`manifest.backend_name`, one of `LLAMACPP_BACKEND_PRIORITY`'s -/// `"rocm"` / `"vulkan"`), rather than scanning every known backend directory in -/// priority order. Lemonade reports the generic `"rocm"` — never the more specific -/// `rocm-stable`/`rocm-nightly` — but may extract the actual build into either -/// versioned directory, so `"rocm"` maps to the whole ROCm family; `"vulkan"` never -/// does. This is what lets alignment verification reject a stale ROCm directory left -/// by an earlier attempt when this round actually picked `vulkan` (or the reverse). +/// The `bin/llamacpp/` directories that may hold the `llama-server` for a manifest's +/// selected backend, or `None` when that backend cannot be served directly. Only +/// `rocm` and `vulkan` are valid manifest backend names: installs record whatever +/// Lemonade selects from its `rocm`/`vulkan` priority list, so the `rocm-stable` and +/// `rocm-nightly` directory names are members of the `rocm` family, never backend +/// names of their own. `cpu` and unknown names fail closed (no CPU fallback). +pub(crate) fn backend_server_dirs(backend_name: &str) -> Option<&'static [&'static str]> { + match backend_name { + ROCM_BACKEND_NAME => Some(&ROCM_LLAMACPP_BACKEND_DIRS), + "vulkan" => Some(&["vulkan"]), + _ => None, + } +} + +/// Find a GPU llama-server scoped to the backend Lemonade selected +/// (`manifest.backend_name`, one of `LLAMACPP_BACKEND_PRIORITY`'s `"rocm"` / +/// `"vulkan"`), rather than scanning every known backend directory. `rocm` accepts +/// its directory family (`rocm-stable`, `rocm-nightly`, `rocm`); Vulkan never crosses +/// over to a stale ROCm install or the reverse. This is what lets alignment +/// verification reject a stale ROCm directory left by an earlier attempt when this +/// round actually picked `vulkan`. CPU and unknown names fail closed. Both flat and +/// one-level build-numbered extraction layouts are supported. pub(crate) fn find_llama_server_binary_for_backend( manifest: &LemonadeInstallManifest, backend_name: &str, ) -> Option { let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); let binary = platform_binary_name("llama-server"); - let family: &[&str] = if backend_name == ROCM_BACKEND_NAME { - &ROCM_LLAMACPP_BACKEND_DIRS - } else { - &["vulkan"] - }; - family + backend_server_dirs(backend_name)? .iter() .find_map(|backend| find_binary_in(&llamacpp_dir.join(backend), &binary)) } +/// Look for `binary` directly in `dir`, then one level down in each subdirectory. +pub(crate) fn find_binary_in(dir: &Path, binary: &str) -> Option { + let direct = dir.join(binary); + if direct.is_file() { + return Some(direct); + } + fs::read_dir(dir) + .ok()? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .map(|subdir| subdir.join(binary)) + .find(|candidate| candidate.is_file()) +} + #[cfg(test)] mod tests { use super::*; @@ -467,4 +518,37 @@ vllm rocm unsupported Requires Linux ); fs::remove_dir_all(&dir).ok(); } + + #[test] + fn find_llama_server_binary_for_backend_fails_closed_for_cpu_and_unknown() { + let dir = scratch_dir("find-server-fail-closed"); + let runtime_dir = dir.join("runtime"); + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + let server = platform_binary_name("llama-server"); + // Every directory holds a server, so a fallback to any of them would be + // found: only failing closed returns None. + for backend in [ + "rocm-stable", + "rocm-nightly", + "rocm", + "vulkan", + "cpu", + "mystery", + ] { + fs::create_dir_all(llamacpp.join(backend)).unwrap(); + fs::write(llamacpp.join(backend).join(&server), b"x").unwrap(); + } + let manifest = test_manifest(runtime_dir); + // `rocm-stable`/`rocm-nightly` are directories of the `rocm` family, never + // backend names a manifest records. + for backend in ["cpu", "mystery", "", "rocm-stable", "rocm-nightly"] { + assert_eq!(backend_server_dirs(backend), None, "{backend:?}"); + assert_eq!( + find_llama_server_binary_for_backend(&manifest, backend), + None, + "{backend:?} must not fall back to another backend's server" + ); + } + fs::remove_dir_all(&dir).ok(); + } } diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs index 6d96eed836..fb49ecccb6 100644 --- a/engines/lemonade/src/process.rs +++ b/engines/lemonade/src/process.rs @@ -53,6 +53,38 @@ pub(crate) fn spawn_lemond( port: u16, log_path: Option<&Path>, process_env: &LemonadeProcessEnvironment, +) -> Result { + spawn_lemond_with_parent( + manifest, + host, + port, + log_path, + process_env, + &ParentRuntimeEnvironment::current(), + LogMode::Fresh, + ) +} + +/// How a spawned `lemond` treats the service log it is given. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum LogMode { + /// Start the log over: this is the serve's own server. + Fresh, + /// Keep what is there and append: a helper `lemond` (the one that pulls a + /// model) shares the log of a server that already wrote the reason it is needed. + Append, +} + +/// `spawn_lemond` with the parent runtime environment supplied by the caller, so +/// tests need not read or create anything under the real user dirs. +pub(crate) fn spawn_lemond_with_parent( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, + log_mode: LogMode, ) -> Result { #[cfg(windows)] if let Some(log_path) = log_path { @@ -76,13 +108,13 @@ pub(crate) fn spawn_lemond( .arg("--port") .arg(port.to_string()) .stdin(Stdio::null()); - apply_lemonade_process_environment(&mut command, process_env)?; + let vars = lemonade_process_environment_vars(process_env, parent)?; + apply_environment_vars(&mut command, &vars); if let Some(log_path) = log_path { - if let Some(parent) = log_path.parent() { - fs::create_dir_all(parent)?; + if let Some(log_dir) = log_path.parent() { + fs::create_dir_all(log_dir)?; } - let log = fs::File::create(log_path) - .with_context(|| format!("failed to create {}", log_path.display()))?; + let log = open_service_log(log_path, log_mode)?; command.stdout(Stdio::from(log.try_clone()?)); command.stderr(Stdio::from(log)); } else { @@ -95,6 +127,23 @@ pub(crate) fn spawn_lemond( .map(LemondChild::Child) } +/// Open the service log in append mode, starting it over first for +/// [`LogMode::Fresh`]. The child writes +/// through this handle while this process appends its own lines (for example the +/// non-ROCm backend notice) through another; without `O_APPEND` the child's next +/// write would land at its own offset, on top of what was appended. +fn open_service_log(log_path: &Path, log_mode: LogMode) -> Result { + if log_mode == LogMode::Fresh { + fs::File::create(log_path) + .with_context(|| format!("failed to create {}", log_path.display()))?; + } + fs::OpenOptions::new() + .append(true) + .create(true) + .open(log_path) + .with_context(|| format!("failed to open {}", log_path.display())) +} + pub(crate) enum LemondChild { Child(std::process::Child), #[cfg(windows)] @@ -137,6 +186,14 @@ pub(crate) struct LemondExitStatus { } impl LemondExitStatus { + #[cfg(test)] + pub(crate) fn for_test(success: bool, description: &str) -> Self { + Self { + success, + description: description.to_owned(), + } + } + pub(crate) const fn success(&self) -> bool { self.success } @@ -215,11 +272,14 @@ pub(crate) fn lemonade_process_environment() -> Result Result<()> { - let vars = lemonade_process_environment_vars(env, &ParentRuntimeEnvironment::current())?; + let vars = lemonade_process_environment_vars(env, parent)?; apply_environment_vars(command, &vars); Ok(()) } @@ -227,7 +287,10 @@ pub(crate) fn apply_lemonade_process_environment( /// Apply an already-resolved variable set. Callers that spawn repeatedly (the /// readiness poll) resolve once and reuse, so preparing the runtime directory /// does not repeat its syscalls on every attempt. -fn apply_environment_vars(command: &mut ProcessCommand, vars: &[(&'static str, OsString)]) { +pub(crate) fn apply_environment_vars( + command: &mut ProcessCommand, + vars: &[(&'static str, OsString)], +) { for (key, value) in vars { command.env(key, value); } @@ -278,13 +341,34 @@ pub(crate) fn wait_for_lemonade_cli_status( timeout: Duration, log_path: Option<&Path>, process_env: &LemonadeProcessEnvironment, +) -> Result<()> { + wait_for_lemonade_cli_status_with_parent( + manifest, + host, + port, + timeout, + log_path, + process_env, + &ParentRuntimeEnvironment::current(), + ) +} + +/// `wait_for_lemonade_cli_status` with the parent runtime environment supplied by +/// the caller, so tests need not read or create anything under the real user dirs. +pub(crate) fn wait_for_lemonade_cli_status_with_parent( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + timeout: Duration, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, ) -> Result<()> { let start = std::time::Instant::now(); let mut last_status = None; // Resolved once: the poll runs twice a second, and resolving also prepares // the child's runtime directory on disk. - let env_vars = - lemonade_process_environment_vars(process_env, &ParentRuntimeEnvironment::current())?; + let env_vars = lemonade_process_environment_vars(process_env, parent)?; while start.elapsed() < timeout { let mut command = ProcessCommand::new(&manifest.lemonade); command @@ -336,12 +420,16 @@ fn summarize_startup_log_tail(log_path: Option<&Path>, limit: usize) -> String { } } +/// Install a llama.cpp backend through the `lemonade` CLI. The parent runtime +/// environment is supplied by the caller, so a test can confine what is read and +/// created. pub(crate) fn run_lemonade_backend_install( manifest: &LemonadeInstallManifest, host: &str, port: u16, backend: &str, process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, ) -> Result<()> { let mut command = backend_install_command( manifest, @@ -349,6 +437,7 @@ pub(crate) fn run_lemonade_backend_install( port, backend, process_env, + parent, OutputConsoles::detect(), hide_child_console_window, )?; @@ -367,12 +456,14 @@ pub(crate) fn run_lemonade_backend_install( /// own output. `consoles` and `hide_window` are parameters so a test can assert /// whether the window is hidden for each console state; the one production /// caller passes the real streams and [`hide_child_console_window`]. +#[allow(clippy::too_many_arguments)] fn backend_install_command( manifest: &LemonadeInstallManifest, host: &str, port: u16, backend: &str, process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, consoles: OutputConsoles, hide_window: impl FnOnce(&mut ProcessCommand), ) -> Result { @@ -388,7 +479,7 @@ fn backend_install_command( .stdin(Stdio::null()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()); - apply_lemonade_process_environment(&mut command, process_env)?; + apply_lemonade_process_environment_with_parent(&mut command, process_env, parent)?; if consoles.child_may_hide_its_console() { hide_window(&mut command); } @@ -410,13 +501,17 @@ fn lemonade_pull_args(host: &str, port: u16, checkpoint_ref: &str) -> Vec, process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, ) -> Result<()> { let mut command = ProcessCommand::new(&manifest.lemonade); command @@ -433,7 +528,8 @@ pub(crate) fn run_lemonade_pull( } else { command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); } - apply_lemonade_process_environment(&mut command, process_env)?; + let vars = lemonade_process_environment_vars(process_env, parent)?; + apply_environment_vars(&mut command, &vars); hide_child_console_window(&mut command); let status = command .status() @@ -444,8 +540,11 @@ pub(crate) fn run_lemonade_pull( Ok(()) } +/// Load a model through the Lemonade CLI. The parent runtime environment is supplied +/// by the caller (production passes `ParentRuntimeEnvironment::current()`), so tests +/// need not read or create anything under the real user dirs. #[allow(clippy::too_many_arguments)] -pub(crate) fn run_lemonade_model_load( +pub(crate) fn run_lemonade_model_load_with_parent( manifest: &LemonadeInstallManifest, host: &str, port: u16, @@ -454,6 +553,7 @@ pub(crate) fn run_lemonade_model_load( engine_recipe: Option<&EngineRecipeHint>, log_path: Option<&Path>, process_env: &LemonadeProcessEnvironment, + parent: &ParentRuntimeEnvironment, ) -> Result<()> { let mut command = ProcessCommand::new(&manifest.lemonade); command @@ -476,7 +576,8 @@ pub(crate) fn run_lemonade_model_load( } else { command.stdout(Stdio::inherit()).stderr(Stdio::inherit()); } - apply_lemonade_process_environment(&mut command, process_env)?; + let vars = lemonade_process_environment_vars(process_env, parent)?; + apply_environment_vars(&mut command, &vars); hide_child_console_window(&mut command); let status = command .status() @@ -492,7 +593,11 @@ pub(crate) fn run_lemonade_model_load( /// since the direct-serve path only runs GPU backends. A Lemonade-router entry carries /// `recipe_options`, so its `llamacpp_backend` must match — this keeps a merely /// registered-but-unloaded model (empty `recipe_options`) from reading as ready. -fn models_payload_has_ready_model(value: &Value, model_ref: &str, backend: &str) -> bool { +pub(crate) fn models_payload_has_ready_model( + value: &Value, + model_ref: &str, + backend: &str, +) -> bool { value .get("data") .or_else(|| value.get("models")) @@ -724,16 +829,25 @@ mod tests { /// whether the window was hidden, along with the command itself. fn backend_install_for(consoles: OutputConsoles) -> (ProcessCommand, bool) { let mut hidden = false; + let root = crate::test_support::scratch_dir("backend-install-command"); + // Confined to a scratch root where a confined parent exists (Linux), so + // building the command creates nothing in the real runtime directory. + #[cfg(target_os = "linux")] + let parent = ParentRuntimeEnvironment::rooted_at(&root); + #[cfg(not(target_os = "linux"))] + let parent = ParentRuntimeEnvironment::current(); let command = backend_install_command( &test_manifest(), "127.0.0.1", 8123, "rocm", &LemonadeProcessEnvironment::default(), + &parent, consoles, |_| hidden = true, ) .expect("the command builds"); + fs::remove_dir_all(&root).ok(); (command, hidden) } @@ -776,6 +890,183 @@ mod tests { } } + #[cfg(target_os = "linux")] + #[test] + fn serving_gpu_model_load_process_error_is_returned() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let executable = dir.join("lemonade"); + let args_path = dir.join("args.txt"); + rocm_core::fake_executable::write_fake_executable(&executable, "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$(dirname \"$0\")/args.txt\"\nprintf 'GPU backend initialization failed\\n' >&2\nexit 23\n").unwrap(); + let manifest = LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: executable, + backend_recipe: crate::LLAMACPP_RECIPE.to_owned(), + backend_name: "vulkan".to_owned(), + installed_at_unix_ms: 0, + }; + let log = dir.join("load.log"); + let parent = ParentRuntimeEnvironment::rooted_at(dir); + let error = run_lemonade_model_load_with_parent( + &manifest, + "127.0.0.1", + 11435, + "Qwen3-0.6B-GGUF", + "vulkan", + None, + Some(&log), + &LemonadeProcessEnvironment::default(), + &parent, + ) + .unwrap_err(); + assert!(error.to_string().contains("Lemonade model load failed")); + assert!(error.to_string().contains("23")); + assert!( + fs::read_to_string(log) + .unwrap() + .contains("GPU backend initialization failed") + ); + let args = fs::read_to_string(args_path).unwrap(); + assert!(args.contains("load\nQwen3-0.6B-GGUF\n--llamacpp\nvulkan\n")); + assert!(!args.contains("cpu")); + } + + #[cfg(target_os = "linux")] + fn fake_pull_manifest(dir: &Path, exit_code: i32) -> LemonadeInstallManifest { + let executable = dir.join("lemonade"); + rocm_core::fake_executable::write_fake_executable( + &executable, + format!( + "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$(dirname \"$0\")/args.txt\"\nprintf 'pull output line\\n'\nprintf 'pull failure detail\\n' >&2\nexit {exit_code}\n" + ), + ) + .unwrap(); + LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: executable, + backend_recipe: crate::LLAMACPP_RECIPE.to_owned(), + backend_name: "vulkan".to_owned(), + installed_at_unix_ms: 0, + } + } + + /// The child's handle and a second appender share one log: a line appended + /// after the child's first write must survive the child's next write. + #[test] + fn a_line_appended_beside_the_childs_log_handle_survives_its_next_write() { + use std::io::Write as _; + let dir = crate::test_support::scratch_dir("service-log-append"); + let log_path = dir.join("serve.log"); + let mut child_handle = open_service_log(&log_path, LogMode::Fresh).unwrap(); + child_handle.write_all(b"child line one\n").unwrap(); + let notice = + crate::device_policy::announce_non_rocm_backend("vulkan", Some(&log_path)).unwrap(); + child_handle.write_all(b"child line two\n").unwrap(); + // The whole file, not a substring: without append the child's second write + // lands on top of the start of the notice and a `contains` on its tail + // would still match. + let log = fs::read_to_string(&log_path).unwrap(); + assert_eq!( + log, + format!("child line one\n{notice}\nchild line two\n"), + "{log}" + ); + fs::remove_dir_all(dir).unwrap(); + } + + /// A helper `lemond` appends to the log of the server whose failure made it + /// necessary; only the serve's own server starts the log over. + #[test] + fn a_helper_server_keeps_the_log_it_is_given() { + use std::io::Write as _; + let dir = crate::test_support::scratch_dir("service-log-modes"); + let log_path = dir.join("serve.log"); + fs::write( + &log_path, + "router failed to load the model +", + ) + .unwrap(); + let mut helper = open_service_log(&log_path, LogMode::Append).unwrap(); + helper + .write_all( + b"helper line +", + ) + .unwrap(); + assert_eq!( + fs::read_to_string(&log_path).unwrap(), + "router failed to load the model +helper line +" + ); + drop(open_service_log(&log_path, LogMode::Fresh).unwrap()); + assert_eq!(fs::read_to_string(&log_path).unwrap(), ""); + fs::remove_dir_all(dir).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn pull_runs_exact_checkpoint_argv() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = fake_pull_manifest(dir, 0); + let log = dir.join("pull.log"); + run_lemonade_pull_with_parent( + &manifest, + "127.0.0.1", + 11436, + "owner/repo:Q4_0", + Some(&log), + &LemonadeProcessEnvironment::default(), + &ParentRuntimeEnvironment::rooted_at(dir), + ) + .unwrap(); + assert_eq!( + fs::read_to_string(dir.join("args.txt")).unwrap(), + "--host\n127.0.0.1\n--port\n11436\npull\nowner/repo:Q4_0\n" + ); + assert!( + fs::read_to_string(log) + .unwrap() + .contains("pull output line") + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn pull_failure_is_returned_with_checkpoint_status_and_logged_output() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let manifest = fake_pull_manifest(dir, 7); + let log = dir.join("pull.log"); + let error = run_lemonade_pull_with_parent( + &manifest, + "127.0.0.1", + 11436, + "owner/repo:Q4_0", + Some(&log), + &LemonadeProcessEnvironment::default(), + &ParentRuntimeEnvironment::rooted_at(dir), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("owner/repo:Q4_0"), "{error}"); + assert!(error.contains('7'), "{error}"); + // The process output is streamed to the log rather than embedded in the error. + assert!( + fs::read_to_string(log) + .unwrap() + .contains("pull failure detail") + ); + } + #[test] fn models_ready_accepts_direct_serve_and_gates_router_backend() { // Direct-serve (stock llama-server): no `recipe_options` → ready by name. diff --git a/engines/lemonade/src/router_serve.rs b/engines/lemonade/src/router_serve.rs new file mode 100644 index 0000000000..ed0df4fe81 --- /dev/null +++ b/engines/lemonade/src/router_serve.rs @@ -0,0 +1,860 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Serving through the Lemonade router (`lemond`): spawn the server, select a +//! GPU llama.cpp backend, load the model by its router name and verify it. +//! +//! The orchestration lives behind [`RouterSteps`] so the state it persists and +//! the way it fails can be tested without a real `lemond`. Production uses +//! [`LemondRouterSteps`]. There is deliberately no direct-server step here: any +//! failure is returned to the caller as an error and the server is stopped. The +//! engine state is then left at `failed`, except for a model load or +//! verification failure: that is tagged [`RouterLoadFailure`] and does not write +//! `failed`, so the caller can decide, on Linux, whether the selected GPU +//! backend can serve the model directly instead and own the final state. + +use anyhow::{Context, Result, anyhow}; +use serde_json::json; +use std::path::Path; +use std::time::Duration; + +use crate::install::LemonadeRuntime; +use crate::process::{LemonadeProcessEnvironment, LemondChild, LemondExitStatus}; +use crate::runtime_dir::ParentRuntimeEnvironment; +use crate::{LLAMACPP_RECIPE, ServeHttpRequest}; + +/// Marks a router model-load or load-verification failure (as opposed to a +/// server that never became ready, a failed backend selection, or a server that +/// exited later). Only these are eligible for the direct-serving fallback. +#[derive(Debug)] +pub(crate) struct RouterLoadFailure(pub(crate) String); + +impl std::fmt::Display for RouterLoadFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// Whether `error` came from the router failing to load or verify the model. +pub(crate) fn is_router_load_failure(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() +} + +/// The side-effecting steps of a router serve. +pub(crate) trait RouterSteps { + fn runtime(&self) -> &LemonadeRuntime; + /// Start `lemond` and return its PID. + fn spawn_server(&mut self) -> Result; + fn wait_ready(&mut self) -> Result<()>; + /// Select (and install if needed) the GPU llama.cpp backend; returns its name. + fn select_backend(&mut self) -> Result; + fn load_model(&mut self, router_model_ref: &str, backend: &str) -> Result<()>; + /// Whether the endpoint reports the model loaded on `backend`. `Err` means the + /// question could not be asked (transport failure, timeout), which is not the + /// same as the endpoint answering that it is not loaded. + fn model_loaded(&mut self, router_model_ref: &str, backend: &str) -> Result; + /// Whether a chat completion succeeds; `Err` as for [`Self::model_loaded`]. + fn chat_smoke(&mut self, router_model_ref: &str) -> Result; + /// Kill and reap the spawned server. + fn stop_server(&mut self); + fn server_start_ticks(&self, pid: u32) -> Option; + /// Block until the server exits. + fn wait_server(&mut self) -> Result; +} + +/// Serve `router_model_ref` through the router. `request.model_ref` keeps the +/// text the user asked for; every router call uses the normalized name. +pub(crate) fn serve_router( + request: &ServeHttpRequest, + steps: &mut S, + router_model_ref: &str, +) -> Result<()> { + let server_pid = steps.spawn_server()?; + // Every step after the spawn is inside this one fallible block, so a failure + // anywhere in it (state writes included) stops the server and marks the state + // `failed`; the server has no `Drop` and would otherwise keep running. + let outcome = (|| -> Result<()> { + crate::state::write_running_state_for_model( + request, + steps.runtime(), + std::process::id(), + Some(server_pid), + "running", + router_model_ref, + )?; + steps + .wait_ready() + .context("Lemonade server did not become ready")?; + let backend = steps + .select_backend() + .context("failed to select a supported Lemonade llama.cpp backend")?; + // A load failure is tagged `RouterLoadFailure`, which is what lets + // `with_direct_fallback` serve a registered model directly on Linux. The + // router's own error is kept and returned when no direct route is open. + steps + .load_model(router_model_ref, &backend) + .map_err(|error| { + error.context(RouterLoadFailure(format!( + "failed to load {} with Lemonade {LLAMACPP_RECIPE}:{backend}", + request.model_ref + ))) + })?; + // Only an answer counts as "not loaded". A probe that could not ask (a + // slow or briefly unreachable endpoint) is a plain failure: it is not a + // `RouterLoadFailure`, so it does not trigger the direct fallback, whose + // download can be several gigabytes, on the strength of a blip. + let verified = steps + .model_loaded(router_model_ref, &backend) + .context("failed to ask Lemonade whether the model is loaded")? + && steps + .chat_smoke(router_model_ref) + .context("failed to run the Lemonade chat smoke test")?; + if !verified { + return Err(anyhow!( + "Lemonade load completed but the endpoint did not report a {LLAMACPP_RECIPE}:{backend}-loaded model" + ) + .context(RouterLoadFailure(format!( + "failed to verify {} with Lemonade {LLAMACPP_RECIPE}:{backend}", + request.model_ref + )))); + } + crate::state::merge_json_state( + &request.state_path, + &json!({ + "status": "ready", + "server_pid": server_pid, + // Identity token for the server PID, captured while the child is alive. + "server_start_ticks": steps.server_start_ticks(server_pid), + "backend_state": "ready", + "backend_requested": backend, + "load_response": { + "status": "loaded", + "method": "lemonade-cli", + "model_name": router_model_ref, + "llamacpp_backend": backend + }, + }), + )?; + // Only now is it true that Lemonade is serving on this backend: the load + // and its verification passed and the `ready` state is written. Announcing + // earlier would claim it for a load that fails, for a ready write that + // fails (the server is then stopped), and again on the fallback. + crate::device_policy::announce_non_rocm_backend(&backend, request.log_path.as_deref()); + Ok(()) + })(); + if let Err(error) = outcome { + steps.stop_server(); + // A load or verification failure may still be served directly, so the + // fallback decides the final state: it sets `starting` when it runs and + // `failed` when it does not (see `with_direct_fallback`). Marking `failed` + // here would let a reader in the gap lock the record at `failed`. + if !is_router_load_failure(&error) { + let _ = crate::state::mark_json_status(&request.state_path, "failed"); + } + return Err(error); + } + let status = match steps.wait_server() { + Ok(status) => status, + Err(error) => { + steps.stop_server(); + let _ = crate::state::mark_json_status(&request.state_path, "failed"); + return Err(error); + } + }; + crate::state::mark_json_status( + &request.state_path, + if status.success() { + "stopped" + } else { + "failed" + }, + )?; + if status.success() { + Ok(()) + } else { + anyhow::bail!("Lemonade server exited with status {status}") + } +} + +/// The real `lemond` and `lemonade` CLI behind [`RouterSteps`]. +pub(crate) struct LemondRouterSteps<'a> { + pub(crate) runtime: LemonadeRuntime, + pub(crate) request: &'a ServeHttpRequest, + pub(crate) process_env: &'a LemonadeProcessEnvironment, + pub(crate) log_path: Option<&'a Path>, + /// The environment the child processes inherit from; tests confine it. + pub(crate) parent: ParentRuntimeEnvironment, + pub(crate) child: Option, +} + +impl RouterSteps for LemondRouterSteps<'_> { + fn runtime(&self) -> &LemonadeRuntime { + &self.runtime + } + + fn spawn_server(&mut self) -> Result { + let child = crate::process::spawn_lemond_with_parent( + &self.runtime.manifest, + &self.request.host, + self.request.port, + self.log_path, + self.process_env, + &self.parent, + crate::process::LogMode::Fresh, + )?; + let pid = child.id(); + self.child = Some(child); + Ok(pid) + } + + fn wait_ready(&mut self) -> Result<()> { + crate::process::wait_for_lemonade_cli_status_with_parent( + &self.runtime.manifest, + &self.request.host, + self.request.port, + Duration::from_secs(30), + self.log_path, + self.process_env, + &self.parent, + ) + } + + fn select_backend(&mut self) -> Result { + crate::llamacpp_backend::ensure_best_llamacpp_backend_with_parent( + &mut self.runtime.manifest, + &self.request.host, + self.request.port, + self.process_env, + false, + &self.parent, + ) + } + + fn load_model(&mut self, router_model_ref: &str, backend: &str) -> Result<()> { + crate::process::run_lemonade_model_load_with_parent( + &self.runtime.manifest, + &self.request.host, + self.request.port, + router_model_ref, + backend, + self.request.engine_recipe.as_ref(), + self.log_path, + self.process_env, + &self.parent, + ) + } + + fn model_loaded(&mut self, router_model_ref: &str, backend: &str) -> Result { + crate::process::query_loaded_model_endpoint( + &crate::state::endpoint_url(&self.request.host, self.request.port), + router_model_ref, + backend, + ) + } + + fn chat_smoke(&mut self, router_model_ref: &str) -> Result { + crate::process::query_chat_smoke_endpoint( + &self.request.host, + self.request.port, + router_model_ref, + ) + } + + fn stop_server(&mut self) { + if let Some(mut child) = self.child.take() { + // The whole tree: a router that failed to load a model can leave a + // backend process holding the device memory, and the fallback starts + // a second server on the same device and port straight after. + let _ = rocm_core::terminate_process_tree(child.id()); + let _ = child.wait(); + } + } + + fn server_start_ticks(&self, pid: u32) -> Option { + rocm_core::process_start_ticks(pid) + } + + fn wait_server(&mut self) -> Result { + self.child + .as_mut() + .context("Lemonade server was not started")? + .wait() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::install::LemonadeInstallManifest; + use std::path::PathBuf; + + #[derive(Clone, Copy, PartialEq, Eq)] + enum Fail { + None, + Ready, + Backend, + Load, + Loaded, + Smoke, + /// The loaded-model probe cannot reach the endpoint. + LoadedUnreachable, + /// The chat smoke probe cannot reach the endpoint. + SmokeUnreachable, + /// The state file cannot be written once the server is spawned. + StateWrite, + /// The state file becomes unwritable between verification and the ready merge. + ReadyMerge, + WaitServer, + FailedExit, + } + + struct FakeSteps { + runtime: LemonadeRuntime, + fail: Fail, + calls: Vec, + state_path: PathBuf, + /// The state file as it stood when the load step ran. + state_at_load: Option, + } + + impl FakeSteps { + fn new(dir: &Path, fail: Fail) -> Self { + Self { + runtime: LemonadeRuntime { + manifest: LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: dir.join("lemonade"), + backend_recipe: LLAMACPP_RECIPE.to_owned(), + backend_name: "vulkan".to_owned(), + installed_at_unix_ms: 0, + }, + }, + fail, + calls: Vec::new(), + state_path: dir.join("state.json"), + state_at_load: None, + } + } + } + + impl RouterSteps for FakeSteps { + fn runtime(&self) -> &LemonadeRuntime { + &self.runtime + } + fn spawn_server(&mut self) -> Result { + self.calls.push("spawn".to_owned()); + Ok(4_000_000) + } + fn wait_ready(&mut self) -> Result<()> { + self.calls.push("ready".to_owned()); + if self.fail == Fail::Ready { + anyhow::bail!("not ready"); + } + Ok(()) + } + fn select_backend(&mut self) -> Result { + self.calls.push("backend".to_owned()); + if self.fail == Fail::Backend { + anyhow::bail!("no backend"); + } + Ok("vulkan".to_owned()) + } + fn load_model(&mut self, router_model_ref: &str, backend: &str) -> Result<()> { + self.calls + .push(format!("load:{router_model_ref}:{backend}")); + self.state_at_load = crate::state::read_service_state(&self.state_path).ok(); + if self.fail == Fail::Load { + anyhow::bail!("load exploded"); + } + Ok(()) + } + fn model_loaded(&mut self, router_model_ref: &str, backend: &str) -> Result { + self.calls + .push(format!("loaded:{router_model_ref}:{backend}")); + if self.fail == Fail::LoadedUnreachable { + anyhow::bail!("connection refused"); + } + Ok(self.fail != Fail::Loaded) + } + fn chat_smoke(&mut self, router_model_ref: &str) -> Result { + self.calls.push(format!("smoke:{router_model_ref}")); + if self.fail == Fail::SmokeUnreachable { + anyhow::bail!("operation timed out"); + } + Ok(self.fail != Fail::Smoke) + } + fn stop_server(&mut self) { + self.calls.push("stop".to_owned()); + } + fn server_start_ticks(&self, _pid: u32) -> Option { + // Asked for while the ready state is assembled, i.e. after every + // verification step and before the merge is written. + if self.fail == Fail::ReadyMerge { + std::fs::remove_file(&self.state_path).unwrap(); + std::fs::create_dir(&self.state_path).unwrap(); + } + None + } + fn wait_server(&mut self) -> Result { + self.calls.push("wait".to_owned()); + if self.fail == Fail::WaitServer { + anyhow::bail!("lost the server"); + } + Ok(if self.fail == Fail::FailedExit { + LemondExitStatus::for_test(false, "exit status: 23") + } else { + LemondExitStatus::for_test(true, "exit status: 0") + }) + } + } + + fn request(dir: &Path) -> ServeHttpRequest { + ServeHttpRequest { + service_id: "test".to_owned(), + model_ref: "qwen-smoke".to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: vec![0], + runtime_id: None, + env_id: None, + state_path: dir.join("state.json"), + log_path: None, + engine_recipe: None, + } + } + + const ROUTER_NAME: &str = "Qwen3-0.6B-GGUF"; + + #[test] + fn router_state_keeps_requested_text_and_records_router_name() { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), Fail::None); + serve_router(&request, &mut steps, ROUTER_NAME).unwrap(); + + // The state written before the load: requested text kept, router name served. + let at_load = steps.state_at_load.expect("state exists before load"); + assert_eq!(at_load["status"], "running"); + assert_eq!(at_load["model_ref"], "qwen-smoke"); + assert_eq!(at_load["canonical_model_id"], ROUTER_NAME); + + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["model_ref"], "qwen-smoke"); + assert_eq!(state["canonical_model_id"], ROUTER_NAME); + assert_eq!(state["load_response"]["model_name"], ROUTER_NAME); + assert_eq!(state["backend_requested"], "vulkan"); + assert_eq!(state["status"], "stopped"); + } + + #[test] + fn router_name_not_alias_reaches_load_loaded_and_smoke_checks() { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), Fail::None); + serve_router(&request, &mut steps, ROUTER_NAME).unwrap(); + assert_eq!( + steps.calls, + [ + "spawn".to_owned(), + "ready".to_owned(), + "backend".to_owned(), + format!("load:{ROUTER_NAME}:vulkan"), + format!("loaded:{ROUTER_NAME}:vulkan"), + format!("smoke:{ROUTER_NAME}"), + "wait".to_owned(), + ] + ); + } + + #[test] + fn router_failures_return_error_stop_server_and_mark_state_failed() { + for (fail, expected, last_call) in [ + (Fail::Ready, "did not become ready", "ready"), + (Fail::Backend, "failed to select a supported", "backend"), + ( + Fail::Load, + "failed to load qwen-smoke with Lemonade", + "load", + ), + ( + Fail::Loaded, + "failed to verify qwen-smoke with Lemonade", + "loaded", + ), + ( + Fail::Smoke, + "failed to verify qwen-smoke with Lemonade", + "smoke", + ), + ] { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), fail); + let error = serve_router(&request, &mut steps, ROUTER_NAME) + .expect_err(expected) + .to_string(); + assert!(error.contains(expected), "{error}"); + // The server is stopped, and the router branch has no other step to + // fall back to: the calls are the router sequence up to the failure, + // then the stop, and nothing after it. + assert_eq!( + steps.calls.last().map(String::as_str), + Some("stop"), + "{:?}", + steps.calls + ); + assert!( + steps.calls[steps.calls.len() - 2].starts_with(last_call), + "{:?}", + steps.calls + ); + assert!(!steps.calls.iter().any(|call| call == "wait")); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + // A load or verification failure is left for the direct fallback to + // settle (it sets `starting` or `failed`); the other failures are final. + let expected_status = if matches!(fail, Fail::Load | Fail::Loaded | Fail::Smoke) { + "running" + } else { + "failed" + }; + assert_eq!(state["status"], expected_status, "{expected}"); + assert_eq!(state["model_ref"], "qwen-smoke"); + } + } + + /// A verification probe that could not reach the endpoint is not an answer + /// that the model is missing: it stops the server and fails the serve, and it + /// is not a `RouterLoadFailure`, so the direct fallback (and its download) + /// does not run on the strength of it. An answer of "not loaded" still is one. + #[test] + fn an_unreachable_verification_probe_is_not_a_load_failure() { + for (fail, probe, last_call) in [ + ( + Fail::LoadedUnreachable, + "whether the model is loaded", + "loaded", + ), + (Fail::SmokeUnreachable, "chat smoke test", "smoke"), + ] { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), fail); + let error = serve_router(&request, &mut steps, ROUTER_NAME).expect_err(probe); + assert!(!is_router_load_failure(&error), "{error:#}"); + assert!(format!("{error:#}").contains(probe), "{error:#}"); + assert_eq!( + steps.calls.last().map(String::as_str), + Some("stop"), + "{:?}", + steps.calls + ); + assert!( + steps.calls[steps.calls.len() - 2].starts_with(last_call), + "{:?}", + steps.calls + ); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed", "{state}"); + } + for fail in [Fail::Loaded, Fail::Smoke] { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), fail); + let error = serve_router(&request, &mut steps, ROUTER_NAME).unwrap_err(); + assert!(is_router_load_failure(&error), "{error:#}"); + } + } + + /// A failure at a step after the spawn (not a router step) must still stop the + /// server it spawned. + fn assert_post_spawn_failure_stops_server(fail: Fail, expected: &str) { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + if fail == Fail::StateWrite { + // A directory where the state file belongs: every write fails. + std::fs::create_dir(&request.state_path).unwrap(); + } + let mut steps = FakeSteps::new(temporary.path(), fail); + let error = format!( + "{:#}", + serve_router(&request, &mut steps, ROUTER_NAME).expect_err(expected) + ); + assert!(error.contains(expected), "{error}"); + // The spawned server is stopped exactly once, and nothing runs after it. + assert_eq!( + steps.calls.last().map(String::as_str), + Some("stop"), + "{:?}", + steps.calls + ); + assert_eq!( + steps.calls.iter().filter(|call| *call == "stop").count(), + 1, + "{:?}", + steps.calls + ); + if fail == Fail::WaitServer { + // The state file is writable here, so the failure is recorded. + let state = crate::state::read_service_state(&request.state_path).unwrap(); + assert_eq!(state["status"], "failed"); + } + } + + #[test] + fn state_write_failure_right_after_spawn_stops_the_server() { + assert_post_spawn_failure_stops_server(Fail::StateWrite, "failed to write"); + } + + #[test] + fn ready_state_merge_failure_stops_the_server() { + assert_post_spawn_failure_stops_server(Fail::ReadyMerge, "failed to write"); + } + + /// The ready write fails after the model loaded and verified, and the server + /// is stopped: the log must not say Lemonade is serving on the backend. + #[test] + fn a_failed_ready_state_write_does_not_announce_a_backend() { + let temporary = tempfile::tempdir().unwrap(); + let mut request = request(temporary.path()); + request.log_path = Some(temporary.path().join("serve.log")); + let mut steps = FakeSteps::new(temporary.path(), Fail::ReadyMerge); + serve_router(&request, &mut steps, ROUTER_NAME).unwrap_err(); + assert_eq!(steps.calls.last().map(String::as_str), Some("stop")); + let log = std::fs::read_to_string(request.log_path.as_ref().unwrap()).unwrap_or_default(); + assert!(!log.contains("not ROCm"), "{log}"); + } + + #[test] + fn wait_server_failure_stops_the_server_and_marks_state_failed() { + assert_post_spawn_failure_stops_server(Fail::WaitServer, "lost the server"); + } + + #[test] + fn only_a_load_or_verification_failure_is_eligible_for_direct_fallback() { + for (fail, eligible) in [ + (Fail::Load, true), + (Fail::Loaded, true), + (Fail::Smoke, true), + (Fail::Ready, false), + (Fail::Backend, false), + (Fail::StateWrite, false), + (Fail::ReadyMerge, false), + (Fail::WaitServer, false), + (Fail::FailedExit, false), + ] { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + if fail == Fail::StateWrite { + std::fs::create_dir(&request.state_path).unwrap(); + } + let mut steps = FakeSteps::new(temporary.path(), fail); + let error = serve_router(&request, &mut steps, ROUTER_NAME).unwrap_err(); + assert_eq!(is_router_load_failure(&error), eligible, "{error:#}"); + // The server was stopped, except when it had already exited by itself. + assert_eq!( + steps.calls.iter().any(|call| call == "stop"), + fail != Fail::FailedExit, + "{:?}", + steps.calls + ); + } + } + + #[test] + fn a_non_rocm_backend_is_named_in_the_notice_and_matches_the_state() { + let temporary = tempfile::tempdir().unwrap(); + let mut request = request(temporary.path()); + request.log_path = Some(temporary.path().join("serve.log")); + let mut steps = FakeSteps::new(temporary.path(), Fail::None); + serve_router(&request, &mut steps, ROUTER_NAME).unwrap(); + let state = crate::state::read_service_state(&request.state_path).unwrap(); + let backend = state["backend_requested"].as_str().unwrap(); + assert_eq!(backend, "vulkan"); + // The notice is computed from the recorded backend, so it cannot disagree. + let notice = crate::device_policy::non_rocm_backend_notice(backend).unwrap(); + assert!( + notice.contains("vulkan") && notice.contains("not ROCm"), + "{notice}" + ); + // A managed serve discards stdout and stderr, so the router path has to + // put the notice in the serve log, where the user can read it. + let log = std::fs::read_to_string(request.log_path.as_ref().unwrap()).unwrap(); + assert!(log.contains(¬ice), "{log}"); + assert_eq!(crate::device_policy::non_rocm_backend_notice("rocm"), None); + assert_eq!( + crate::device_policy::non_rocm_backend_notice("rocm-stable"), + None + ); + } + + /// The notice says Lemonade is serving on the backend, so it is written only + /// once the model is loaded and verified: a failed load or verification claims + /// nothing (and the fallback that may follow announces for itself). + #[test] + fn a_failed_load_or_verification_does_not_announce_a_backend() { + for fail in [Fail::Load, Fail::Loaded, Fail::Smoke] { + let temporary = tempfile::tempdir().unwrap(); + let mut request = request(temporary.path()); + request.log_path = Some(temporary.path().join("serve.log")); + let mut steps = FakeSteps::new(temporary.path(), fail); + serve_router(&request, &mut steps, ROUTER_NAME).unwrap_err(); + let log = + std::fs::read_to_string(request.log_path.as_ref().unwrap()).unwrap_or_default(); + assert!(!log.contains("not ROCm"), "{log}"); + } + } + + #[test] + fn nonzero_server_exit_returns_error_and_marks_state_failed() { + let temporary = tempfile::tempdir().unwrap(); + let request = request(temporary.path()); + let mut steps = FakeSteps::new(temporary.path(), Fail::FailedExit); + let error = serve_router(&request, &mut steps, ROUTER_NAME).unwrap_err(); + assert!( + error + .to_string() + .contains("Lemonade server exited with status exit status: 23") + ); + assert_eq!( + crate::state::read_service_state(&request.state_path).unwrap()["status"], + "failed" + ); + assert_eq!(steps.calls.last().map(String::as_str), Some("wait")); + } + + #[cfg(target_os = "linux")] + #[test] + fn real_router_stop_kills_and_reaps_its_server() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let request = request(dir); + let env = LemonadeProcessEnvironment::default(); + let mut steps = LemondRouterSteps { + runtime: LemonadeRuntime { + manifest: crate::test_http::fake_runtime::manifest(dir, 0, None), + }, + request: &request, + process_env: &env, + log_path: None, + parent: ParentRuntimeEnvironment::rooted_at(dir), + child: None, + }; + let pid = steps.spawn_server().unwrap(); + let process = PathBuf::from(format!("/proc/{pid}")); + assert!(process.exists(), "the real spawn must create a process"); + steps.stop_server(); + // Clean up even if a no-op stop mutation left the child alive. + let gone = !process.exists(); + if !gone { + let _ = crate::state::terminate_pid(pid, true); + if let Some(mut child) = steps.child.take() { + let _ = child.wait(); + } + } + assert!( + gone, + "stop_server must terminate and reap the spawned lemond" + ); + assert!(steps.child.is_none()); + } + + /// Backend selection runs `lemonade backends` through the serve request's + /// confined parent environment, never the host's: the child must see the + /// confined runtime directory, and the answer must come from its listing. + #[cfg(target_os = "linux")] + #[test] + fn real_backend_selection_runs_under_the_confined_parent() { + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + let request = request(dir); + let env = LemonadeProcessEnvironment::default(); + let mut manifest = crate::test_http::fake_runtime::manifest(dir, 0, None); + manifest.lemonade = dir.join("lemonade-backends"); + rocm_core::fake_executable::write_fake_executable( + &manifest.lemonade, + r#"#!/bin/sh +printf '%s' "$XDG_RUNTIME_DIR" > "$(dirname "$0")/xdg.txt" +printf 'Recipe Backend Status Message Action +llamacpp vulkan installed ok none +' +"#, + ) + .unwrap(); + let mut steps = LemondRouterSteps { + runtime: LemonadeRuntime { manifest }, + request: &request, + process_env: &env, + log_path: None, + parent: ParentRuntimeEnvironment::rooted_at(dir), + child: None, + }; + assert_eq!(steps.select_backend().unwrap(), "vulkan"); + assert_eq!( + std::fs::read_to_string(dir.join("xdg.txt")).unwrap(), + dir.join("xdg-runtime").to_string_lossy() + ); + // The listing said `installed`, so nothing was installed. + assert_eq!(steps.runtime.manifest.backend_name, "vulkan"); + } + + #[cfg(target_os = "linux")] + #[test] + fn real_steps_send_the_router_name_not_the_requested_alias() { + use crate::test_http::{Reply, fake_runtime, spawn_fixture}; + let temporary = tempfile::tempdir().unwrap(); + let dir = temporary.path(); + // A name the alias `qwen-smoke` does not normalize to, so a step that used + // the requested text could not pass by matching it. + let router_name = "Registered-Model-GGUF"; + let (port, server) = spawn_fixture(Duration::from_secs(10), move |recorded| { + let chat = recorded.method == "POST"; + Reply { + status: 200, + body: if chat { + json!({"choices": [{"message": {"content": "ok"}}]}) + } else { + json!({"all_models_loaded": [{"model_name": router_name, "recipe_options": {"llamacpp_backend": "vulkan"}}]}) + } + .to_string(), + last: chat, + } + }); + let mut request = request(dir); + request.port = port; + assert_eq!(request.model_ref, "qwen-smoke"); + let env = LemonadeProcessEnvironment::default(); + let mut steps = LemondRouterSteps { + runtime: LemonadeRuntime { + manifest: fake_runtime::manifest(dir, 0, None), + }, + request: &request, + process_env: &env, + log_path: None, + parent: ParentRuntimeEnvironment::rooted_at(dir), + child: None, + }; + steps.load_model(router_name, "vulkan").unwrap(); + assert!(steps.model_loaded(router_name, "vulkan").unwrap()); + assert!(steps.chat_smoke(router_name).unwrap()); + + let load = fake_runtime::calls(dir); + assert_eq!( + load, + [format!( + "--host 127.0.0.1 --port {port} load {router_name} --llamacpp vulkan --save-options" + )] + ); + let seen = server.join().unwrap(); + let chat = seen.iter().find(|r| r.method == "POST").expect("chat sent"); + assert_eq!(chat.path, "/v1/chat/completions"); + assert_eq!( + serde_json::from_str::(&chat.body).unwrap()["model"], + router_name + ); + } +} diff --git a/engines/lemonade/src/runtime_dir.rs b/engines/lemonade/src/runtime_dir.rs index 44b021c9b5..cc9ff29706 100644 --- a/engines/lemonade/src/runtime_dir.rs +++ b/engines/lemonade/src/runtime_dir.rs @@ -42,6 +42,22 @@ impl ParentRuntimeEnvironment { } } +// Used by Linux-only fake-executable tests throughout the engine; +// a plain `cfg(test)` leaves it unused (and dead-code denied) on other targets. +#[cfg(all(test, target_os = "linux"))] +impl ParentRuntimeEnvironment { + /// A parent environment confined to `root`: it already names a runtime + /// directory, so resolving it creates nothing and reads no real user dirs. + pub(crate) fn rooted_at(root: &Path) -> Self { + Self { + xdg_runtime_dir: Some(root.join("xdg-runtime").into_os_string()), + home: Some(root.join("home").into_os_string()), + user: Some("test".to_owned()), + temp_dir: root.join("tmp"), + } + } +} + /// Leaf directory the engine gets inside a fallback runtime root, so a crashing /// or misbehaving server cannot disturb the telemetry socket that shares the /// same root. diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index b4b3323be6..56ca91cf06 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -55,6 +55,24 @@ pub(crate) fn write_running_state( pid: u32, server_pid: Option, status: &str, +) -> Result<()> { + write_running_state_for_model( + request, + runtime, + pid, + server_pid, + status, + &request.model_ref, + ) +} + +pub(crate) fn write_running_state_for_model( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + pid: u32, + server_pid: Option, + status: &str, + served_model_ref: &str, ) -> Result<()> { write_state( &request.state_path, @@ -65,7 +83,7 @@ pub(crate) fn write_running_state( "pid": pid, "server_pid": server_pid, "model_ref": request.model_ref, - "canonical_model_id": request.model_ref, + "canonical_model_id": served_model_ref, "host": request.host, "port": request.port, "endpoint_url": endpoint_url(&request.host, request.port), diff --git a/engines/lemonade/src/test_http.rs b/engines/lemonade/src/test_http.rs new file mode 100644 index 0000000000..fc48bcfc70 --- /dev/null +++ b/engines/lemonade/src/test_http.rs @@ -0,0 +1,163 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A test-owned loopback HTTP fixture shared by the engine's tests. + +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +/// One request the fixture received. +#[derive(Debug, Clone)] +pub(crate) struct Recorded { + pub(crate) method: String, + pub(crate) path: String, + pub(crate) body: String, +} + +/// What the fixture does with a request: the HTTP status and JSON body to answer +/// with, and whether that was the last request it should serve. +pub(crate) struct Reply { + pub(crate) status: u16, + pub(crate) body: String, + pub(crate) last: bool, +} + +/// Serve loopback HTTP until `respond` marks a reply as the last one or `deadline` +/// passes. Returns the port and a handle yielding every request received. +pub(crate) fn spawn_fixture( + deadline: Duration, + respond: impl Fn(&Recorded) -> Reply + Send + 'static, +) -> (u16, JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + listener.set_nonblocking(true).unwrap(); + let handle = std::thread::spawn(move || { + let mut seen = Vec::new(); + let stop_at = Instant::now() + deadline; + while Instant::now() < stop_at { + let Ok((mut stream, _)) = listener.accept() else { + std::thread::sleep(Duration::from_millis(10)); + continue; + }; + // An accepted socket can inherit the listener's non-blocking mode + // (Windows does), which would make the first read fail with WouldBlock. + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let Some(recorded) = read_request(&mut stream) else { + continue; + }; + let reply = respond(&recorded); + seen.push(recorded); + let _ = write!( + stream, + "HTTP/1.1 {} X\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + reply.status, + reply.body.len(), + reply.body + ); + if reply.last { + break; + } + } + seen + }); + (port, handle) +} + +fn read_request(stream: &mut impl Read) -> Option { + let mut bytes = Vec::new(); + let mut buffer = [0; 1024]; + let (header_end, content_len) = loop { + let count = stream.read(&mut buffer).ok().filter(|count| *count > 0)?; + bytes.extend_from_slice(&buffer[..count]); + if let Some(end) = bytes.windows(4).position(|part| part == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&bytes[..end]); + let len = headers + .lines() + .find_map(|line| { + let (key, value) = line.split_once(':')?; + key.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0); + break (end + 4, len); + } + }; + while bytes.len() < header_end + content_len { + let count = stream.read(&mut buffer).ok().filter(|count| *count > 0)?; + bytes.extend_from_slice(&buffer[..count]); + } + let head = String::from_utf8_lossy(&bytes[..header_end]).into_owned(); + let mut request_line = head.lines().next()?.split_whitespace(); + Some(Recorded { + method: request_line.next()?.to_owned(), + path: request_line.next()?.to_owned(), + body: String::from_utf8_lossy(&bytes[header_end..header_end + content_len]).into_owned(), + }) +} + +/// Fake `lemond`/`lemonade` executables for the engine's tests. The fake CLI appends +/// one line per invocation to `calls.txt` beside it (`cat` it to see the exact argv), +/// and answers `pull` by dropping a GGUF into the hub cache under `cache/`. On a +/// pull it also copies the service `state.json` beside it to `state-at-pull.json`, +/// so a test can read the engine state as it was while the pull ran. +#[cfg(target_os = "linux")] +pub(crate) mod fake_runtime { + use crate::install::LemonadeInstallManifest; + use std::path::Path; + + pub(crate) fn write_executable(path: &Path, script: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + rocm_core::fake_executable::write_fake_executable(path, script).unwrap(); + } + + /// A manifest whose `lemond` idles until killed and whose `lemonade` records its + /// argv. `pull_exit` is the exit status of `lemonade pull`; on success the pull + /// also creates `pulled_gguf` (relative to `dir/cache/huggingface/hub`), if given. + pub(crate) fn manifest( + dir: &Path, + pull_exit: i32, + pulled_gguf: Option<&str>, + ) -> LemonadeInstallManifest { + write_executable(&dir.join("lemond"), "#!/bin/sh\nexec sleep 30\n"); + let create = pulled_gguf.map_or_else(String::new, |relative| { + let target = dir.join("cache/huggingface/hub").join(relative); + format!( + "mkdir -p '{}' && : > '{}'\n", + target.parent().unwrap().display(), + target.display() + ) + }); + write_executable( + &dir.join("lemonade"), + &format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$(dirname \"$0\")/calls.txt\"\nfor arg in \"$@\"; do\n if [ \"$arg\" = pull ]; then\n cp \"$(dirname \"$0\")/state.json\" \"$(dirname \"$0\")/state-at-pull.json\" 2>/dev/null\n {create} exit {pull_exit}\n fi\ndone\nexit 0\n" + ), + ); + LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: dir.join("lemonade"), + backend_recipe: crate::LLAMACPP_RECIPE.to_owned(), + backend_name: "vulkan".to_owned(), + installed_at_unix_ms: 0, + } + } + + /// The `lemonade` invocations so far, one argv per line. + pub(crate) fn calls(dir: &Path) -> Vec { + std::fs::read_to_string(dir.join("calls.txt")) + .unwrap_or_default() + .lines() + .map(str::to_owned) + .collect() + } +} diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 830135e12c..feddea5300 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -191,6 +191,16 @@ reason = "Lemonade serving fails on the Strix Halo Ubuntu lanes, so the model ne serve_timeout_secs = 90 flaky = true +# The dashboard journey does the same real lemonade serve as serve-lemonade-inference, +# so on a Linux lemonade host it is just as intermittent. The bounded serve timeout +# makes a stuck serve fail this one scenario instead of stalling the whole lane. +[["dash-lemonade-managed-inference-journey"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "ROCMAI-898" +reason = "The managed Lemonade serve on Linux is intermittent: it can reach ready and then shut down, or never become ready, so the journey cannot be relied on to complete." +serve_timeout_secs = 90 +flaky = true + # --- EAI-7223: chat requests with tool definitions rejected (vLLM path) --- [["chat-tool-definitions-accepted"]] when = { effective_engine = "vllm" } diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index ed695dd0b7..42b174c0a3 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -75,7 +75,13 @@ Feature: Interactive dashboard When the user opens the dashboard And the user opens the Observe view Then the managed model is displayed - When the user quits the dashboard + When the user opens the managed services overlay + Then the Services overlay lists the managed model with its port + When the user asks to stop the listed service + Then a stop approval prompt names the managed model and its port + When the user declines the approval prompt + And the user closes the managed services overlay + And the user quits the dashboard Then the dashboard exits successfully @@ -910,3 +916,59 @@ Feature: Interactive dashboard Then the CPU graph and the per-core strip sit side by side When the user quits the dashboard Then the dashboard exits successfully + + @id:dash-lemonade-managed-inference-journey @requires-os:linux @requires-gpu @requires-engine:lemonade + Scenario: dash-52 - A real managed Lemonade GPU model responds to inference and appears in the dashboard + Given a managed runtime is active + And the dashboard watches only managed services + And a GGUF model is being served on lemonade + Then the model responds to inference requests + When the user opens the dashboard + And the user opens the Observe view + Then the real managed Lemonade model is displayed + When the user opens instance detail + Then the managed model's instance detail shows it ready on the port it was served on + When the user closes instance detail + And the user selects engine status in the Serving view + Then Lemonade is shown as a managed runtime installation + When the user quits the dashboard + Then the dashboard exits successfully + And the model responds to inference requests + + @id:dash-lemonade-external-protocol-discovery @requires-os:linux + Scenario: dash-53 - External Lemonade protocol discovery displays every loaded model without adopting the service + Given an external Lemonade server has two models loaded + When the user opens the dashboard + And the user opens the Observe view + Then both loaded models are displayed with throughput shown as unknown + When the user opens the managed services overlay + Then the Services overlay has no managed servers + When the user attempts to stop and restart the external models + Then the Services overlay closes without an approval prompt or job + And both loaded models are still displayed + And the external server received no lifecycle request and no managed record exists + When the user selects engine status in the Serving view + Then Lemonade is shown as externally running without a managed installation + When the user quits the dashboard + Then the dashboard exits successfully + And the external server received no lifecycle request and no managed record exists + + @id:dash-lemonade-external-protocol-transitions @requires-os:linux + Scenario: dash-54 - External Lemonade protocol models disappear on unload, return on reload, and disappear again when the server stops responding + Given an external Lemonade server has two models loaded + When the user opens the dashboard + And the user opens the Observe view + Then both loaded models are displayed with throughput shown as unknown + When the external Lemonade server unloads its second model + Then only the first model is displayed, with its throughput + When the external Lemonade server unloads every model + Then no loaded model is displayed and none are reported as discovered + When the external Lemonade server loads both models again + Then both loaded models are displayed with throughput shown as unknown + And the external server received no lifecycle request and no managed record exists + When the external Lemonade server stops responding + Then no loaded model is displayed and none are reported as discovered + When the user selects engine status in the Serving view + Then Lemonade returns to the managed installation status without external availability + When the user quits the dashboard + Then the dashboard exits successfully diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index e8deb5fd4d..3623c1e80a 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -79,14 +79,11 @@ Feature: Model serving Then the response contains a model reply And the response identifies the correct model - # HF-checkpoint direct-serve canary (EAI-8026). An `owner/repo:variant` ref - # bypasses Lemonade's model router and runs a packaged llama-server directly - # (`serve_hf_checkpoint`), which bails explicitly if no backend binary is found - # under bin/llamacpp// — unlike the short-recipe-name path above, whose - # managed-lemonade fallback would mask the identical failure behind the - # ROCMAI-898 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as - # `serve-lemonade-inference` (cache-shared, no extra download) so this stays a - # fast per-PR canary rather than needing the @nightly large-checkpoint path. + # Explicit HF-checkpoint canary: `owner/repo:variant` runs the selected GPU + # llama-server directly and fails if that backend binary is missing. Named + # catalog entries can also serve directly on Linux when their metadata is + # simple enough; other registered entries use Lemonade's router. Both canaries + # use the same small checkpoint, sharing the cache without another download. @id:serve-hf-checkpoint-inference @requires-gpu @requires-engine:lemonade Scenario: serve-08 - A canonical Hugging Face checkpoint serves and responds to inference Given a managed runtime is active @@ -906,3 +903,40 @@ Feature: Model serving And the user is told the model needs an explicit GGUF variant And no quantization is suggested for a repo the CLI has not inspected And no local server was recorded + # The pinned Lemonade catalog's smallest vision GGUF needs its mmproj file. + # Serving its registered name must retain the router, not flatten the entry + # into a single-checkpoint direct serve. Use a scenario-owned port and service; + # the additional model download stays off the ordinary per-PR path. + # Linux only: registered names take the router on Windows both with and + # without the direct llama-server route, so a Windows run cannot tell the + # two behaviours apart. + # No expectations row is added for this scenario. The flaky Linux rows cover + # the direct llama-server route, and this scenario takes the router route, so + # those rows' failure mode does not describe it. It is nightly-only. + @id:serve-lemonade-complex-registered-inference @requires-gpu @requires-os:linux @requires-engine:lemonade @nightly + Scenario: serve-57 - A registered vision GGUF retains the Lemonade router and managed lifecycle + Given a managed runtime is active + And a registered vision GGUF is being served on lemonade on its own port + Then the managed engine state confirms the Lemonade router loaded the registered model + When the user sends a chat completion request + Then the response contains a model reply + And the response identifies the correct model + When the user stops the scenario's managed Lemonade service + Then the managed service is stopped and its endpoint is closed + + # Which runtime a registered name takes is user-observable: a simple catalog + # entry (one checkpoint with a variant, no extra files) is served directly by + # the selected GPU llama-server on Linux, while + # `serve-lemonade-complex-registered-inference` covers the router route for an + # entry that needs more. The endpoint answering would not tell the + # two apart, so this reads the engine state's method and backend. + @id:serve-lemonade-simple-registered-direct-route @requires-gpu @requires-os:linux @requires-engine:lemonade + Scenario: serve-58 - A simple registered GGUF name is served directly by the selected GPU backend + Given a managed runtime is active + And a simple registered GGUF is being served on lemonade on its own port + Then the managed engine state records direct serving by the selected GPU backend + When the user sends a chat completion request + Then the response contains a model reply + And the response identifies the correct model + When the user stops the scenario's managed Lemonade service + Then the managed service is stopped and its endpoint is closed diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index c800a4fcb2..86be40ec51 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -346,11 +346,19 @@ Feature: Runtime configuration # environment and has no llama.cpp backend to align; `@nightly` for the same # reason as the vLLM torch-alignment scenarios above -- a real managed SDK # and a real backend download, not something to repeat on every PR. + # + # The advice `rocm serve` prints for a missing llama-server is `rocm engines + # install lemonade --reinstall`. The Givens install Lemonade and delete the + # selected backend's llama-server, so the last Then can only pass if the + # reinstall put it back. @id:runtime-lemonade-backend-alignment-reported @requires-gpu @requires-os:windows @requires-engine:lemonade @nightly Scenario: runtime-16 - Reinstalling Lemonade reports whether its ROCm backend was aligned Given a managed runtime is active + And the lemonade engine is installed + And the selected Lemonade backend's llama-server binary has been deleted When the user reinstalls the lemonade engine Then the CLI reports that Lemonade's ROCm backend was aligned to the active SDK + And the selected Lemonade backend's llama-server binary is installed # `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT` is the exit for a hand-edited # `backend_versions.json` -- the alignment runs on every Lemonade install, so diff --git a/tests/e2e-cucumber/src/dash_screen.rs b/tests/e2e-cucumber/src/dash_screen.rs new file mode 100644 index 0000000000..eb8a0c6c2a --- /dev/null +++ b/tests/e2e-cucumber/src/dash_screen.rs @@ -0,0 +1,301 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Pure matching over captured dashboard screens and config files. +//! +//! The cucumber binary has no unit-test harness, so the logic that the +//! dashboard scenarios depend on lives here, where a failing case can be shown. + +use serde_json::{Map, Value}; + +/// Fewest cells after the model cell of an Observe instances row. +/// +/// The cells are `TOK/S TOK/W TTFT TPOT POWER QUEUE KV%` (`instances.rs`). A +/// cell with a unit (`12.00 tok/W`) splits into more than one, so a live row +/// can have more. +pub const MIN_OBSERVE_METRIC_CELLS: usize = 7; + +/// The cells after the model cell of an Observe instances row whose model +/// starts with `model_prefix`. Free text that merely mentions the model, or a +/// row that was cut short, does not match. +/// +/// The row carries no status or port. Use [`instance_detail_matches`] for those. +pub fn observe_row_cells<'a>(line: &'a str, model_prefix: &str) -> Option> { + let mut cells = line.split_whitespace(); + cells.find(|cell| cell.starts_with(model_prefix))?; + let rest: Vec<&str> = cells.collect(); + (rest.len() >= MIN_OBSERVE_METRIC_CELLS).then_some(rest) +} + +/// The port of an endpoint URL such as `http://127.0.0.1:11435/v1`. +pub fn endpoint_port(endpoint: &str) -> Option { + let authority = endpoint.split("://").nth(1)?.split('/').next()?; + let port = authority.rsplit_once(':')?.1; + (!port.is_empty() && port.chars().all(|c| c.is_ascii_digit())).then(|| port.to_string()) +} + +/// True when the screen holds a matching instance-detail summary. +/// +/// The status badge must be `status` with `port: ` on the same line, and +/// a `model:` cell must be followed by a name starting with `model_prefix` +/// (`render_summary` in `instance_detail.rs`). +pub fn instance_detail_matches(screen: &str, model_prefix: &str, status: &str, port: &str) -> bool { + // Panel borders can touch a label (`│model:`), so treat them as separators. + let screen = strip_box_drawing(screen); + let screen = screen.as_str(); + let status_and_port = screen.lines().any(|line| { + let cells: Vec<&str> = line.split_whitespace().collect(); + cells.contains(&status) + && cells + .windows(2) + .any(|pair| pair[0] == "port:" && pair[1] == port) + }); + let model = screen.lines().any(|line| { + let mut cells = line.split_whitespace(); + cells.any(|cell| cell == "model:") + && cells + .next() + .is_some_and(|name| name.starts_with(model_prefix)) + }); + status_and_port && model +} + +/// Replace the box-drawing characters of the terminal UI with spaces. +fn strip_box_drawing(screen: &str) -> String { + screen + .chars() + .map(|c| { + if ('\u{2500}'..='\u{257f}').contains(&c) { + ' ' + } else { + c + } + }) + .collect() +} + +/// Set only the `dashboard.daemon.*` keys of `update` on `existing`, leaving +/// every other key (an active runtime, for example) as it was. +/// +/// # Panics +/// Panics when `update` has no `dashboard.daemon` object; that is a bug in the +/// calling step, not a runtime condition. +pub fn merge_dashboard_daemon_config(existing: Option, update: &Value) -> Value { + let mut root = match existing { + Some(Value::Object(map)) => map, + _ => Map::new(), + }; + let updates = update + .pointer("/dashboard/daemon") + .and_then(Value::as_object) + .expect("update must carry dashboard.daemon keys"); + let dashboard = object_entry(&mut root, "dashboard"); + let daemon = object_entry(dashboard, "daemon"); + for (key, value) in updates { + daemon.insert(key.clone(), value.clone()); + } + Value::Object(root) +} + +/// Turn Lemonade discovery off unless the config already says otherwise. +/// +/// The dashboard daemon probes `127.0.0.1:13305` by default. A scenario that +/// does not exercise discovery must not pick up a real Lemonade server that +/// happens to listen there, so the isolated config starts with discovery off. +/// A scenario that wants discovery writes `enable_lemonade` itself, and that +/// value is kept. +pub fn with_lemonade_discovery_default_off(existing: Option) -> Value { + if existing + .as_ref() + .and_then(|config| config.pointer("/dashboard/daemon/enable_lemonade")) + .is_some() + { + return existing.expect("checked above"); + } + merge_dashboard_daemon_config( + existing, + &serde_json::json!({ "dashboard": { "daemon": { "enable_lemonade": false } } }), + ) +} + +/// The object stored under `key`, replacing a missing or non-object value. +fn object_entry<'a>(parent: &'a mut Map, key: &str) -> &'a mut Map { + let slot = parent + .entry(key.to_string()) + .or_insert_with(|| Value::Object(Map::new())); + if !slot.is_object() { + *slot = Value::Object(Map::new()); + } + slot.as_object_mut().expect("slot is an object") +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const MANAGED_ROW: &str = + "Qwen3-0.6B-GGUF — — — — — — —"; + const DETAIL: &str = "│ READY id: svc-1 port: 11435 tp: 1 │\n\ + │ model: Qwen3-0.6B-GGUF gpus: 0 tok/W: — │"; + + #[test] + fn observe_row_matches_a_full_managed_row() { + let cells = observe_row_cells(MANAGED_ROW, "Qwen3-0.6B").expect("row matches"); + assert_eq!(cells.len(), MIN_OBSERVE_METRIC_CELLS); + let with_units = "Qwen3-0.6B-GGUF 41.2 tok/s 0.52 tok/W 50ms 12ms 80W 0/0 3%"; + assert!(observe_row_cells(with_units, "Qwen3-0.6B").is_some()); + } + + #[test] + fn observe_row_rejects_other_model_truncated_row_and_prose() { + let other = MANAGED_ROW.replace("Qwen3-0.6B-GGUF", "Phi-3-mini"); + assert!(observe_row_cells(&other, "Qwen3-0.6B").is_none()); + let truncated = "Qwen3-0.6B-GGUF — — —"; + assert!(observe_row_cells(truncated, "Qwen3-0.6B").is_none()); + let prose = "loaded Qwen3-0.6B-GGUF on the default port"; + assert!(observe_row_cells(prose, "Qwen3-0.6B").is_none()); + } + + #[test] + fn endpoint_port_reads_the_serving_port() { + assert_eq!( + endpoint_port("http://127.0.0.1:11435/v1").as_deref(), + Some("11435") + ); + assert_eq!(endpoint_port("http://localhost/v1"), None); + assert_eq!(endpoint_port("not a url"), None); + } + + #[test] + fn detail_matches_ready_managed_model_on_the_served_port() { + assert!(instance_detail_matches( + DETAIL, + "Qwen3-0.6B", + "READY", + "11435" + )); + } + + /// Captured from a real run: the panel border touches the `model:` label. + const REAL_DETAIL: &str = "│ ╭─╮Node╭─╮Instance · lemonade-qwen3-0-6b-gguf-1792015518018╭───╮──────╮ │\n\ + │ │ │ READY id: lemonade-qwen3-0-6b-gguf-1792015518018 port: 11435 tp: 0 │ │ │\n\ + │ │ — │model: Qwen3-0.6B-GGUF gpus: - tok/W: - gen: - │ │ │\n\ + │ │ │partition: - quantization: - vram: 0 / 0 MiB │ │ │"; + + #[test] + fn detail_matches_the_screen_a_real_run_rendered() { + assert!(instance_detail_matches( + REAL_DETAIL, + "Qwen3-0.6B", + "READY", + "11435" + )); + assert!(!instance_detail_matches( + REAL_DETAIL, + "Qwen3-0.6B", + "READY", + "11436" + )); + assert!(!instance_detail_matches( + &REAL_DETAIL.replace("READY", "STARTING"), + "Qwen3-0.6B", + "READY", + "11435" + )); + } + + #[test] + fn detail_rejects_wrong_status_port_or_model() { + let starting = DETAIL.replace("READY", "STARTING"); + assert!(!instance_detail_matches( + &starting, + "Qwen3-0.6B", + "READY", + "11435" + )); + let stopped = DETAIL.replace("READY", "STOPPED"); + assert!(!instance_detail_matches( + &stopped, + "Qwen3-0.6B", + "READY", + "11435" + )); + let other_port = DETAIL.replace("11435", "8000"); + assert!(!instance_detail_matches( + &other_port, + "Qwen3-0.6B", + "READY", + "11435" + )); + let other_model = DETAIL.replace("Qwen3-0.6B-GGUF", "Phi-3-mini"); + assert!(!instance_detail_matches( + &other_model, + "Qwen3-0.6B", + "READY", + "11435" + )); + // A status word elsewhere on screen does not stand in for the badge on + // the port line. + let elsewhere = "READY\n│ id: svc-1 port: 11435 tp: 1 │\n│ model: Qwen3-0.6B-GGUF │"; + assert!(!instance_detail_matches( + elsewhere, + "Qwen3-0.6B", + "READY", + "11435" + )); + } + + #[test] + fn dashboard_config_merge_keeps_other_keys() { + let existing = json!({ + "runtime": { "active": "therock-7.1" }, + "dashboard": { "theme": "dark", "daemon": { "lemonade_port": 1 } } + }); + let update = json!({ + "dashboard": { "daemon": { "enable_lemonade": false } } + }); + let merged = merge_dashboard_daemon_config(Some(existing), &update); + assert_eq!(merged["runtime"]["active"], "therock-7.1"); + assert_eq!(merged["dashboard"]["theme"], "dark"); + assert_eq!(merged["dashboard"]["daemon"]["lemonade_port"], 1); + assert_eq!(merged["dashboard"]["daemon"]["enable_lemonade"], false); + } + + #[test] + fn lemonade_discovery_defaults_off_and_keeps_other_keys() { + let existing = json!({ "runtime": { "active": "therock-7.1" } }); + for start in [None, Some(existing.clone()), Some(json!("garbage"))] { + let merged = with_lemonade_discovery_default_off(start); + assert_eq!(merged["dashboard"]["daemon"]["enable_lemonade"], false); + } + let merged = with_lemonade_discovery_default_off(Some(existing)); + assert_eq!(merged["runtime"]["active"], "therock-7.1"); + } + + #[test] + fn lemonade_discovery_default_keeps_an_explicit_choice() { + for enabled in [true, false] { + let existing = json!({ + "dashboard": { "daemon": { "enable_lemonade": enabled, "lemonade_port": 9 } } + }); + let merged = with_lemonade_discovery_default_off(Some(existing)); + assert_eq!(merged["dashboard"]["daemon"]["enable_lemonade"], enabled); + assert_eq!(merged["dashboard"]["daemon"]["lemonade_port"], 9); + } + } + + #[test] + fn dashboard_config_merge_starts_from_nothing_or_garbage() { + let update = json!({ "dashboard": { "daemon": { "lemonade_port": 7 } } }); + for existing in [ + None, + Some(json!("not an object")), + Some(json!({"dashboard": 3})), + ] { + let merged = merge_dashboard_daemon_config(existing, &update); + assert_eq!(merged["dashboard"]["daemon"]["lemonade_port"], 7); + } + } +} diff --git a/tests/e2e-cucumber/src/lemonade_backend.rs b/tests/e2e-cucumber/src/lemonade_backend.rs new file mode 100644 index 0000000000..1cf5f53885 --- /dev/null +++ b/tests/e2e-cucumber/src/lemonade_backend.rs @@ -0,0 +1,167 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Where an installed Lemonade runtime keeps the `llama-server` of a backend. +//! +//! The layout mirrors what the Lemonade engine searches when it serves a model +//! directly: `bin/llamacpp//llama-server` under the runtime +//! directory, either flat or one build-numbered subdirectory down. The `rocm` +//! backend is a family of directory names; Vulkan never crosses over to ROCm. + +use std::path::{Path, PathBuf}; + +/// Directory names that hold the `rocm` backend, in lookup order. +const ROCM_DIRS: [&str; 3] = ["rocm-stable", "rocm-nightly", "rocm"]; + +/// The outcome of looking for a backend's `llama-server`. +#[derive(Debug, PartialEq, Eq)] +pub struct ServerLookup { + /// The binary, when one was found. + pub found: Option, + /// Every directory that was searched, in order. + pub searched: Vec, +} + +/// The platform's file name for the `llama-server` binary. +const fn server_binary_name() -> &'static str { + if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + } +} + +/// The directory names a backend may extract into, or `None` for a backend that +/// has no GPU `llama-server` (CPU and unknown names). +pub fn backend_dirs(backend_name: &str) -> Option<&'static [&'static str]> { + match backend_name { + "rocm" => Some(&ROCM_DIRS), + "vulkan" => Some(&["vulkan"]), + _ => None, + } +} + +/// Look for the selected backend's `llama-server` under `runtime_dir`. +/// +/// Callers may rely on whether a binary exists, not on which one is returned when +/// several build-numbered subdirectories each hold one: the engine takes the first +/// in directory order, this returns the first in name order. +/// +/// Returns `Err` for a backend that has no GPU `llama-server`, so a recorded CPU +/// or unknown backend cannot pass as "nothing to find". +pub fn find_llama_server(runtime_dir: &Path, backend_name: &str) -> Result { + let dirs = backend_dirs(backend_name).ok_or_else(|| { + format!("backend `{backend_name}` has no GPU llama-server directory to look in") + })?; + let llamacpp = runtime_dir.join("bin").join("llamacpp"); + let binary = server_binary_name(); + let mut searched = Vec::new(); + for dir in dirs { + let dir = llamacpp.join(dir); + searched.push(dir.clone()); + if let Some(found) = binary_in(&dir, binary) { + return Ok(ServerLookup { + found: Some(found), + searched, + }); + } + } + Ok(ServerLookup { + found: None, + searched, + }) +} + +/// `binary` directly in `dir`, or one level down in a subdirectory. +fn binary_in(dir: &Path, binary: &str) -> Option { + let direct = dir.join(binary); + if direct.is_file() { + return Some(direct); + } + let mut subdirs: Vec = std::fs::read_dir(dir) + .ok()? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .collect(); + // Sorted only so the result is deterministic; which build wins is not part of + // the contract (see `find_llama_server`). + subdirs.sort(); + subdirs + .into_iter() + .map(|subdir| subdir.join(binary)) + .find(|candidate| candidate.is_file()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn install(runtime: &Path, relative: &str) -> PathBuf { + let dir = runtime.join("bin").join("llamacpp").join(relative); + std::fs::create_dir_all(&dir).expect("create backend dir"); + let binary = dir.join(server_binary_name()); + std::fs::write(&binary, b"x").expect("write binary"); + binary + } + + #[test] + fn finds_a_flat_binary_for_the_selected_backend() { + let root = tempfile::tempdir().expect("tempdir"); + let binary = install(root.path(), "vulkan"); + let lookup = find_llama_server(root.path(), "vulkan").expect("lookup"); + assert_eq!(lookup.found, Some(binary)); + } + + #[test] + fn finds_a_build_numbered_binary_in_the_rocm_family() { + let root = tempfile::tempdir().expect("tempdir"); + let binary = install(root.path(), "rocm-stable/llama-b9752"); + let lookup = find_llama_server(root.path(), "rocm").expect("lookup"); + assert_eq!(lookup.found, Some(binary)); + } + + #[test] + fn rocm_lookup_follows_family_order() { + let root = tempfile::tempdir().expect("tempdir"); + install(root.path(), "rocm"); + let stable = install(root.path(), "rocm-stable"); + let lookup = find_llama_server(root.path(), "rocm").expect("lookup"); + assert_eq!(lookup.found, Some(stable)); + } + + #[test] + fn a_different_backend_does_not_stand_in() { + let root = tempfile::tempdir().expect("tempdir"); + install(root.path(), "vulkan"); + install(root.path(), "cpu"); + let lookup = find_llama_server(root.path(), "rocm").expect("lookup"); + assert_eq!(lookup.found, None); + let llamacpp = root.path().join("bin").join("llamacpp"); + let expected: Vec = ROCM_DIRS.iter().map(|dir| llamacpp.join(dir)).collect(); + assert_eq!(lookup.searched, expected); + } + + #[test] + fn a_directory_without_the_binary_is_not_a_hit() { + let root = tempfile::tempdir().expect("tempdir"); + std::fs::create_dir_all(root.path().join("bin/llamacpp/vulkan/llama-b1")) + .expect("create dir"); + let lookup = find_llama_server(root.path(), "vulkan").expect("lookup"); + assert_eq!(lookup.found, None); + assert_eq!(lookup.searched.len(), 1); + } + + #[test] + fn cpu_and_unknown_backends_are_rejected() { + let root = tempfile::tempdir().expect("tempdir"); + install(root.path(), "cpu"); + for backend in ["cpu", "unknown", ""] { + assert!( + find_llama_server(root.path(), backend).is_err(), + "{backend}" + ); + } + } +} diff --git a/tests/e2e-cucumber/src/lemonade_protocol.rs b/tests/e2e-cucumber/src/lemonade_protocol.rs new file mode 100644 index 0000000000..5ab4ab6f5b --- /dev/null +++ b/tests/e2e-cucumber/src/lemonade_protocol.rs @@ -0,0 +1,153 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! HTTP protocol fixture, not a real Lemonade runtime or inference engine. +//! Runs on its own thread so blocking CLI subprocesses cannot stall its routes. + +use std::sync::{Arc, Mutex}; + +use axum::{ + Json, Router, + extract::{Request, State}, + middleware::{self, Next}, + response::Response, + routing::get, +}; +use serde_json::{Value, json}; + +use crate::http_server::{self, ServerHandle}; + +#[derive(Debug)] +pub struct LemonadeProtocolServer { + server: ServerHandle, + loaded: Arc>>, + stray: Arc>>, +} + +impl LemonadeProtocolServer { + pub fn start(models: &[&str]) -> Self { + let loaded = Arc::new(Mutex::new(models.iter().map(|s| (*s).to_owned()).collect())); + let stray = Arc::new(Mutex::new(Vec::new())); + let app = Router::new() + .route("/api/v1/health", get(health)) + .route( + "/api/v1/stats", + get(|| async { + Json(json!({ + "tokens_per_second": 37.5, + "time_to_first_token": 0.05, + "input_tokens": 8, + "output_tokens": 16 + })) + }), + ) + .with_state(Arc::clone(&loaded)) + .layer(middleware::from_fn_with_state( + Arc::clone(&stray), + record_stray, + )); + Self { + server: http_server::spawn_on_own_thread(app), + loaded, + stray, + } + } + + pub const fn port(&self) -> u16 { + self.server.port() + } + + pub fn base_url(&self) -> String { + self.server.base_url() + } + + /// Every request that was not a `GET` of the health or stats route, as + /// `" "`. A client that unloads, loads, or stops a model on + /// this server, or probes any other route, shows up here. + pub fn stray_requests(&self) -> Vec { + self.stray.lock().expect("fixture state poisoned").clone() + } + + pub fn set_loaded(&self, models: &[&str]) { + *self.loaded.lock().expect("fixture state poisoned") = + models.iter().map(|s| (*s).to_owned()).collect(); + } +} + +async fn record_stray( + State(stray): State>>>, + request: Request, + next: Next, +) -> Response { + let method = request.method().clone(); + let path = request.uri().path().to_owned(); + let allowed = method == axum::http::Method::GET + && matches!(path.as_str(), "/api/v1/health" | "/api/v1/stats"); + if !allowed { + stray + .lock() + .expect("fixture state poisoned") + .push(format!("{method} {path}")); + } + next.run(request).await +} + +async fn health(State(loaded): State>>>) -> Json { + let loaded = loaded.lock().expect("fixture state poisoned"); + Json(json!({ + "status": "ok", + "model_loaded": loaded.first(), + "all_models_loaded": loaded + .iter() + .map(|model| json!({"model_name": model})) + .collect::>() + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn records_requests_other_than_get_health_and_stats() { + let fixture = LemonadeProtocolServer::start(&["a", "b"]); + let base = fixture.base_url(); + let client = reqwest::Client::new(); + client + .get(format!("{base}/api/v1/health")) + .send() + .await + .unwrap(); + client + .get(format!("{base}/api/v1/stats")) + .send() + .await + .unwrap(); + assert!(fixture.stray_requests().is_empty()); + + client + .post(format!("{base}/api/v1/unload")) + .send() + .await + .unwrap(); + client + .get(format!("{base}/api/v1/models")) + .send() + .await + .unwrap(); + client + .delete(format!("{base}/api/v1/health")) + .send() + .await + .unwrap(); + assert_eq!( + fixture.stray_requests(), + vec![ + "POST /api/v1/unload", + "GET /api/v1/models", + "DELETE /api/v1/health" + ] + ); + } +} diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index a89e72dd29..b1508b5c64 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -5,13 +5,16 @@ pub mod binary_path; pub mod capability; pub mod chat_request; +pub mod dash_screen; pub mod device_memory; pub mod expectation; pub mod fixture_binary; pub mod harness_diagnostics; pub mod http_server; pub mod installer_fixture; +pub mod lemonade_backend; pub mod lemonade_diagnostics; +pub mod lemonade_protocol; pub mod loopback_http; pub mod mock_server; pub mod model_id; diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index 90d7573ecc..b990259855 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -27,6 +27,21 @@ //! deleted with the scenario. use std::path::Path; +use std::time::Duration; + +/// Extra time the `rocm serve` process gets beyond the model readiness timeout. +/// +/// The CLI's own readiness wait starts only after its pre-launch work (backend +/// and weights fetch), while the harness clock starts at spawn. Bounding the +/// process by the bare timeout would kill it first and lose the CLI's own +/// diagnostic in a slow cold start. The margin still bounds a stuck serve. +pub const SERVE_CLI_MARGIN_SECS: u64 = 300; + +/// How long to let a `rocm serve` process run: the readiness timeout plus +/// [`SERVE_CLI_MARGIN_SECS`]. Model readiness polling keeps the plain timeout. +pub const fn serve_cli_budget(timeout_secs: u64) -> Duration { + Duration::from_secs(timeout_secs.saturating_add(SERVE_CLI_MARGIN_SECS)) +} /// How many trailing lines of a stalled service's log to quote in a failure. const DEFAULT_TAIL_LINES: usize = 40; @@ -348,11 +363,21 @@ pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { #[cfg(test)] mod tests { use super::{ - MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, engine_from_serve_args, - parse_log_path, read_clamped, read_clamped_file, reported_engine, serve_attempt_report, - service_log_tail, slugify, tail_lines, + MAX_ARCHIVED_LOG_BYTES, SERVE_CLI_MARGIN_SECS, ServeAttempt, archive_service_log, + engine_from_serve_args, parse_log_path, read_clamped, read_clamped_file, reported_engine, + serve_attempt_report, serve_cli_budget, service_log_tail, slugify, tail_lines, }; + #[test] + fn cli_budget_outlasts_the_readiness_timeout_by_a_bounded_margin() { + for timeout in [90, 2400] { + let budget = serve_cli_budget(timeout).as_secs(); + assert!(budget > timeout, "the CLI must outlive the readiness wait"); + assert_eq!(budget, timeout + SERVE_CLI_MARGIN_SECS); + } + assert_eq!(serve_cli_budget(u64::MAX).as_secs(), u64::MAX); + } + const SERVE_PLAN: &str = "\ serve plan requested model: Qwen/Qwen3.5-0.8B diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 296bb51ef5..6e8db6c579 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -58,6 +58,7 @@ mod e2e { #[derive(Debug, cucumber::World)] pub struct E2eWorld { pub mock: Option, + pub lemonade_protocol: Option, /// Loopback file server used by artifact-prefetch scenarios. Kept on the /// World so it remains alive while the real `rocmd` subprocess downloads. pub artifact_server: Option, @@ -363,6 +364,7 @@ impl Default for E2eWorld { Self { mock: None, + lemonade_protocol: None, artifact_server: None, paced_download_server: None, comfyui_port_listener: None, @@ -781,6 +783,7 @@ impl Drop for E2eWorld { if let Some(mock) = self.mock.take() { mock.stop(); } + self.lemonade_protocol.take(); self.artifact_server.take(); self.paced_download_server.take(); // A scenario that ran `rocm serve --managed` left a DETACHED supervisor + @@ -958,6 +961,67 @@ pub fn run_rocm(world: &E2eWorld, args: &[&str]) -> (String, String, i32) { ) } +/// Like [`run_rocm`], but gives up after `timeout` instead of blocking forever. +/// +/// [`run_rocm`] uses `Command::output`, which waits for the pipes to close. A +/// `rocm serve --managed` that leaves a supervisor holding an inherited pipe, or +/// that never returns, therefore stalls the whole lane with no report. Here the +/// child writes to files, so an inherited handle cannot keep us waiting, and the +/// child is killed once the deadline passes. A timeout is reported as rc `-1` +/// with a note appended to stderr. +pub fn run_rocm_bounded( + world: &E2eWorld, + args: &[&str], + timeout: std::time::Duration, +) -> (String, String, i32) { + use std::process::Stdio; + + let binary = rocm_binary(); + let dir = tempfile::tempdir().expect("temp dir for bounded command output"); + let out_path = dir.path().join("stdout"); + let err_path = dir.path().join("stderr"); + let mut cmd = std::process::Command::new(&binary); + cmd.args(args); + world.isolate_cmd(&mut cmd); + cmd.stdin(Stdio::null()); + cmd.stdout(Stdio::from( + std::fs::File::create(&out_path).expect("stdout file"), + )); + cmd.stderr(Stdio::from( + std::fs::File::create(&err_path).expect("stderr file"), + )); + let mut child = cmd + .spawn() + .unwrap_or_else(|e| panic!("failed to run {binary}: {e}")); + let deadline = std::time::Instant::now() + timeout; + let mut timed_out = false; + let status = loop { + if let Some(status) = child.try_wait().expect("wait on rocm") { + break Some(status); + } + if std::time::Instant::now() >= deadline { + timed_out = true; + let _ = child.kill(); + let _ = child.wait(); + break None; + } + std::thread::sleep(std::time::Duration::from_millis(200)); + }; + let rc = status.and_then(|s| s.code()).unwrap_or(-1); + let stdout = String::from_utf8_lossy(&std::fs::read(&out_path).unwrap_or_default()).to_string(); + let mut stderr = + String::from_utf8_lossy(&std::fs::read(&err_path).unwrap_or_default()).to_string(); + if timed_out { + stderr = format!( + "{stderr}\n`rocm {}` did not return within {}s and was killed\n", + args.join(" "), + timeout.as_secs() + ); + } + record_command(world.current_scenario.as_deref(), args, rc, &stdout); + (stdout, stderr, rc) +} + /// Run `rocm`, returning stdout, and panic with the full diagnostic bundle /// ([`cli_failure_report`]) if it exits non-zero. /// diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 22eef1e841..ddb5b598d9 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -100,6 +100,330 @@ async fn wait_for_all_markers(tui: &mut TuiSession, what: &str, markers: &[&str] .await } +// This is a protocol contract journey: CLI, embedded daemon and TUI are real; +// the HTTP service is a named protocol fixture, not GPU inference. +const PROTOCOL_MODELS: [&str; 2] = ["Protocol-Qwen-4B", "Protocol-Phi-3B"]; + +#[given("an external Lemonade server has two models loaded")] +async fn external_lemonade_protocol(world: &mut E2eWorld) { + let server = e2e_cucumber::lemonade_protocol::LemonadeProtocolServer::start(&PROTOCOL_MODELS); + configure_lemonade_discovery(world, server.port()); + let root = world.isolated_root.as_ref().expect("isolated root").path(); + assert!( + !root.join("data/services").exists(), + "no managed service may be planted" + ); + world.lemonade_protocol = Some(server); +} + +fn configure_lemonade_discovery(world: &E2eWorld, port: u16) { + // Discovery is off by default in the isolated config (see + // `disable_lemonade_discovery_by_default`), so these scenarios turn it on + // explicitly. `lemonade_host` is omitted to exercise its serde default. + // Socket isolation comes from the PTY driver's HOME/XDG environment. + write_dashboard_config( + world, + serde_json::json!({ + "dashboard": { "daemon": { "enable_lemonade": true, "lemonade_port": port, + "discovery_tick_secs": 0.1, "instance_tick_secs": 0.1 } } + }), + ); +} + +#[given("the dashboard watches only managed services")] +async fn dashboard_managed_only(world: &mut E2eWorld) { + // An unrelated live endpoint on the default port must not supply the + // model or external-running status in this real managed-service journey. + write_dashboard_config( + world, + serde_json::json!({"dashboard": {"daemon": {"enable_lemonade": false}}}), + ); +} + +/// Keep a dashboard that is not testing Lemonade discovery away from any real +/// Lemonade server on the default port. Call before every dashboard spawn. +fn disable_lemonade_discovery_by_default(world: &E2eWorld) { + let root = world.isolated_root.as_ref().expect("isolated root").path(); + let dir = root.join("config"); + std::fs::create_dir_all(&dir).expect("create config dir"); + let path = dir.join("config.json"); + let existing = std::fs::read(&path) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()); + let config = e2e_cucumber::dash_screen::with_lemonade_discovery_default_off(existing); + std::fs::write( + &path, + serde_json::to_vec(&config).expect("serialize config"), + ) + .expect("write config"); +} + +fn write_dashboard_config(world: &E2eWorld, config: serde_json::Value) { + let root = world.isolated_root.as_ref().expect("isolated root").path(); + let dir = root.join("config"); + std::fs::create_dir_all(&dir).expect("create config dir"); + let path = dir.join("config.json"); + // An earlier step may have written the active runtime here; keep it. + let existing = std::fs::read(&path) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()); + let merged = e2e_cucumber::dash_screen::merge_dashboard_daemon_config(existing, &config); + std::fs::write( + &path, + serde_json::to_vec(&merged).expect("serialize config"), + ) + .expect("write config"); +} + +#[then("both loaded models are displayed with throughput shown as unknown")] +async fn external_models_visible(world: &mut E2eWorld) { + session(world) + .wait_for_screen_where( + "both protocol models, with unknown unattributable throughput", + |screen| { + PROTOCOL_MODELS.iter().all(|model| { + screen + .lines() + .any(|line| protocol_tps_cell(line, model) == Some("—")) + }) + }, + default_timeout(), + ) + .await + .expect("external models should be discovered"); +} + +fn protocol_tps_cell<'a>(line: &'a str, model: &str) -> Option<&'a str> { + let mut cells = line.split_whitespace().skip_while(|cell| *cell != model); + cells.next()?; + cells.next() +} + +#[when("the external Lemonade server unloads its second model")] +async fn external_single_model(world: &mut E2eWorld) { + world + .lemonade_protocol + .as_ref() + .expect("fixture") + .set_loaded(&PROTOCOL_MODELS[..1]); +} + +#[then("only the first model is displayed, with its throughput")] +async fn external_single_model_rate(world: &mut E2eWorld) { + session(world) + .wait_for_screen_where( + "single protocol model with unambiguous last-request throughput", + |screen| { + !screen.contains(PROTOCOL_MODELS[1]) + && screen + .lines() + .any(|line| protocol_tps_cell(line, PROTOCOL_MODELS[0]) == Some("38")) + // The rate is the last completed request's, and the screen says so. + && screen.contains("last req tok/s") + }, + default_timeout(), + ) + .await + .expect("single loaded model owns endpoint stats"); +} + +#[when("the external Lemonade server unloads every model")] +async fn external_unload(world: &mut E2eWorld) { + world + .lemonade_protocol + .as_ref() + .expect("fixture") + .set_loaded(&[]); +} + +#[then("no loaded model is displayed and none are reported as discovered")] +async fn external_models_absent(world: &mut E2eWorld) { + session(world) + .wait_for_screen_where( + "unloaded external models disappear", + |screen| { + PROTOCOL_MODELS.iter().all(|m| !screen.contains(m)) + && screen.contains("no serving models discovered") + }, + default_timeout(), + ) + .await + .expect("stale external models must disappear"); +} + +#[when("the external Lemonade server loads both models again")] +async fn external_reload(world: &mut E2eWorld) { + world + .lemonade_protocol + .as_ref() + .expect("fixture") + .set_loaded(&PROTOCOL_MODELS); +} + +#[when("the external Lemonade server stops responding")] +async fn external_unreachable(world: &mut E2eWorld) { + drop(world.lemonade_protocol.take()); +} + +#[then("the external server received no lifecycle request and no managed record exists")] +async fn external_server_untouched(world: &mut E2eWorld) { + // What this proves: the fixture recorded only observational requests (the + // stray-request check below) and the registry holds no record. The health + // read is a sanity check that the fixture still answers; the fixture is an + // in-process thread the dashboard cannot stop, so it is not evidence of + // anything on its own. + let fixture = world.lemonade_protocol.as_ref().expect("fixture"); + let response = reqwest::get(format!("{}/api/v1/health", fixture.base_url())) + .await + .expect("dashboard quit must leave external service alive"); + assert!(response.status().is_success()); + let body: serde_json::Value = response.json().await.expect("health JSON"); + assert_eq!(body["status"], "ok"); + // The dashboard may only observe: any load, unload, stop or other request + // reaching the external server would be a lifecycle action on a service it + // does not own. + assert_eq!( + fixture.stray_requests(), + Vec::::new(), + "the dashboard sent a non-observational request to the external server" + ); + assert_eq!( + body["all_models_loaded"].as_array().expect("models").len(), + 2 + ); + let services = world + .isolated_root + .as_ref() + .expect("root") + .path() + .join("data/services"); + assert!( + !services.exists() + || std::fs::read_dir(services) + .expect("services") + .next() + .is_none(), + "discovery must not adopt an external fixture as a managed service" + ); +} + +#[when("the user selects engine status in the Serving view")] +async fn engine_status_pane(world: &mut E2eWorld) { + let tui = session(world); + tui.use_detail_size().expect("enlarge dashboard"); + tui.send_until("3", "● Serving", default_timeout()) + .await + .expect("Serving view"); + tui.send("gj").expect("select Engines action"); +} + +#[then("Lemonade is shown as externally running without a managed installation")] +async fn external_installation_truth(world: &mut E2eWorld) { + session(world) + .wait_for_screen_where( + "external versus managed engine state", + |screen| { + screen.contains("Lemonade — externally running") + && screen.contains("Managed runtime — not installed") + }, + default_timeout(), + ) + .await + .expect("engine status must distinguish external service and managed install"); +} + +#[then("Lemonade returns to the managed installation status without external availability")] +async fn external_availability_cleared(world: &mut E2eWorld) { + session(world) + .wait_for_screen_where( + "unreachable endpoint no longer counts as external availability", + |screen| { + screen.contains("Lemonade — managed runtime not installed") + && !screen.contains("Lemonade — externally running") + }, + default_timeout(), + ) + .await + .expect("engine status must return to managed inventory after endpoint loss"); +} + +#[then("the real managed Lemonade model is displayed")] +async fn real_lemonade_visible(world: &mut E2eWorld) { + // The preceding real inference assertion identifies this scenario's model. + // Direct llama-server need not expose Lemonade stats: missing throughput is + // truthful and must not cause us to invent a rate to satisfy this journey. + // + // Anchored to a full Observe instances row (the model cell followed by the + // TOK/S .. KV% cells), not to the name appearing anywhere on screen, so a + // stray mention elsewhere cannot satisfy it. The dashboard watches only + // managed services in this scenario, so that row is the registry-owned one. + // The row carries neither status nor port; the next steps read those from + // the instance detail. + session(world) + .wait_for_screen_where( + "an Observe row for the managed Lemonade model", + |screen| { + screen.lines().any(|line| { + e2e_cucumber::dash_screen::observe_row_cells(line, "Qwen3-0.6B").is_some() + }) + }, + default_timeout(), + ) + .await + .expect("dashboard must display this scenario's real served model"); +} + +#[then("the managed model's instance detail shows it ready on the port it was served on")] +async fn real_lemonade_detail_ready(world: &mut E2eWorld) { + let endpoint = world + .endpoint + .clone() + .expect("the serve step sets an endpoint"); + let port = e2e_cucumber::dash_screen::endpoint_port(&endpoint) + .unwrap_or_else(|| panic!("no port in served endpoint {endpoint}")); + session(world) + .wait_for_screen_where( + "instance detail showing the model READY on the served port", + |screen| { + e2e_cucumber::dash_screen::instance_detail_matches( + screen, + "Qwen3-0.6B", + "READY", + &port, + ) + }, + default_timeout(), + ) + .await + .unwrap_or_else(|e| { + panic!("instance detail must show Qwen3-0.6B READY on port {port}: {e}") + }); +} + +#[when("the user closes instance detail")] +async fn close_instance_detail(world: &mut E2eWorld) { + // Esc closes the popup, but with no popup open it opens the menu. So send + // it once and wait for the popup to be gone. + let tui = session(world); + tui.send("\x1b") + .unwrap_or_else(|e| panic!("failed to send Esc: {e}")); + tui.wait_for_screen_where( + "instance detail closed", + |screen| !screen.contains("Instance · "), + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("instance detail did not close: {e}")); +} + +#[then("Lemonade is shown as a managed runtime installation")] +async fn real_lemonade_installed(world: &mut E2eWorld) { + session(world) + .wait_for_screen("Lemonade — managed runtime installed", default_timeout()) + .await + .expect("managed engine Detect evidence must reach dashboard"); +} + // ── Given ────────────────────────────────────────────────────────── #[given("interactive chat uses an offline assistant")] @@ -208,6 +532,7 @@ async fn replay_missing_recording(world: &mut E2eWorld) { )); let _ = std::fs::remove_file(&missing); let path = missing.to_string_lossy().to_string(); + disable_lemonade_discovery_by_default(world); let session = TuiSession::spawn(world, &["dash", "--replay", &path]) .unwrap_or_else(|e| panic!("failed to spawn dash under a pty: {e}")); world.tui = Some(session); @@ -217,6 +542,7 @@ async fn replay_missing_recording(world: &mut E2eWorld) { async fn open_dashboard_demo(world: &mut E2eWorld) { // `--demo` replays a deterministic synthetic session, so the dashboard // populates with no GPU and no daemon — a stable, mock-tier target. + disable_lemonade_discovery_by_default(world); let session = TuiSession::spawn(world, &["dash", "--demo"]) .unwrap_or_else(|e| panic!("failed to open the dashboard: {e}")); world.tui = Some(session); @@ -402,6 +728,7 @@ fn replay_recorded_snapshots_with_host( std::fs::write(&recording, recording_text) .unwrap_or_else(|e| panic!("failed to write {}: {e}", recording.display())); let path = recording.to_string_lossy().to_string(); + disable_lemonade_discovery_by_default(world); let session = TuiSession::spawn(world, &["dash", "--replay", &path]) .unwrap_or_else(|e| panic!("failed to open the dashboard replaying a recording: {e}")); world.tui = Some(session); @@ -424,11 +751,14 @@ async fn open_dashboard_replaying_warning(world: &mut E2eWorld) { } /// One recorded instance per lifecycle state the Services manager colors. The -/// model name carries the state so the scenario can find each row by it. +/// model name carries the state so the scenario can find each row by it. Each is +/// recorded as managed, because the Services manager lists only instances the +/// daemon marked as registry-owned (a recording without the field replays as +/// unmanaged and would leave the overlay empty). fn recorded_instance(id: &str, model: &str, status_json: &str) -> String { format!( concat!( - r#"{{"container_id":"{id}","container_name":"vllm-{id}","status":{status},"#, + r#"{{"container_id":"{id}","container_name":"vllm-{id}","managed":true,"status":{status},"#, r#""model_name":"{model}","gpu_ids":["0"],"tensor_parallel_size":1,"#, r#""vram_used_mb":0,"vram_total_mb":0,"gen_tps":null,"launch_args":[],"env_vars":{{}}}}"#, ), @@ -511,6 +841,7 @@ async fn open_dashboard_in_scenario_folder(world: &mut E2eWorld) { child.display() ) }); + disable_lemonade_discovery_by_default(world); let session = TuiSession::spawn_in(world, &["dash", "--demo"], &root) .unwrap_or_else(|e| panic!("failed to open the dashboard in {}: {e}", root.display())); world.tui = Some(session); @@ -555,6 +886,7 @@ async fn open_dashboard_in_whitespace_named_folder(world: &mut E2eWorld) { cwd.display() ) }); + disable_lemonade_discovery_by_default(world); let session = TuiSession::spawn_in(world, &["dash", "--demo"], &cwd) .unwrap_or_else(|e| panic!("failed to open the dashboard in {}: {e}", cwd.display())); world.tui = Some(session); @@ -567,6 +899,10 @@ async fn open_chat(world: &mut E2eWorld) { } else { &["chat"] }; + // `rocm chat` starts the embedded daemon too, and discovery is on by default. + if world.isolated_root.is_some() { + disable_lemonade_discovery_by_default(world); + } let session = TuiSession::spawn(world, args) .unwrap_or_else(|e| panic!("failed to open interactive chat: {e}")); world.tui = Some(session); @@ -590,6 +926,7 @@ async fn machine_with_no_gpu(world: &mut E2eWorld) { #[when("the user opens the dashboard")] async fn open_dashboard(world: &mut E2eWorld) { + disable_lemonade_discovery_by_default(world); let tui = TuiSession::spawn(world, &["dash"]) .unwrap_or_else(|e| panic!("failed to open the dashboard: {e}")); world.tui = Some(tui); @@ -599,6 +936,7 @@ async fn open_dashboard(world: &mut E2eWorld) { async fn open_launcher(world: &mut E2eWorld) { // Bare `rocm` (no subcommand) opens the launcher front door under an // interactive terminal — the PTY slave satisfies `interactive_terminal()`. + disable_lemonade_discovery_by_default(world); let tui = TuiSession::spawn(world, &[]) .unwrap_or_else(|e| panic!("failed to open the launcher: {e}")); world.tui = Some(tui); @@ -656,6 +994,104 @@ async fn open_services_overlay(world: &mut E2eWorld) { .unwrap_or_else(|e| panic!("failed to open the services overlay: {e}")); } +#[then("the Services overlay has no managed servers")] +async fn external_services_are_read_only(world: &mut E2eWorld) { + session(world) + .wait_for_screen( + "No managed services. Start one with `rocm serve --managed`.", + default_timeout(), + ) + .await + .expect("discovered external models must not become lifecycle targets"); +} + +#[then("the Services overlay lists the managed model with its port")] +async fn services_overlay_lists_managed_model(world: &mut E2eWorld) { + let model = world.model_name.clone().expect("no model name set"); + let port = + e2e_cucumber::dash_screen::endpoint_port(world.endpoint.as_deref().expect("no endpoint")) + .expect("endpoint has a port"); + let port_cell = format!(":{port}"); + session(world) + .wait_for_screen_where( + "a Services row for the managed model and its port", + |screen| { + !screen.contains("No managed services.") + && screen + .lines() + .any(|line| line.contains(&model) && line.contains(&port_cell)) + }, + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("the managed service is not listed in the overlay: {e}")); +} + +#[when("the user asks to stop the listed service")] +async fn user_stages_stop(world: &mut E2eWorld) { + // Sent once: the key stages an approval, and a second copy would land on the + // approval prompt rather than the list. + session(world) + .send("s") + .unwrap_or_else(|e| panic!("failed to send the stop key: {e}")); +} + +#[then("a stop approval prompt names the managed model and its port")] +async fn stop_approval_is_shown(world: &mut E2eWorld) { + let model = world.model_name.clone().expect("no model name set"); + let port = + e2e_cucumber::dash_screen::endpoint_port(world.endpoint.as_deref().expect("no endpoint")) + .expect("endpoint has a port"); + let detail = format!("model: {model} port: {port}"); + session(world) + .wait_for_screen_where( + "a stop approval for the managed service", + |screen| { + screen.contains("Review: stop service") + && screen.contains("Approve (y)") + && screen.contains(&detail) + }, + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("no stop approval prompt appeared: {e}")); +} + +#[when("the user declines the approval prompt")] +async fn user_declines_approval(world: &mut E2eWorld) { + // Esc declines without confirming. The prompt itself going away is the + // proof the key was read; the overlay list stays open behind it. + let tui = session(world); + tui.send("\u{1b}") + .unwrap_or_else(|e| panic!("failed to decline the approval: {e}")); + tui.wait_until_gone("Review: stop service", default_timeout()) + .await + .unwrap_or_else(|e| panic!("the approval prompt did not close: {e}")); +} + +#[when("the user attempts to stop and restart the external models")] +async fn external_lifecycle_keys_do_nothing(world: &mut E2eWorld) { + let tui = session(world); + tui.send("sr").expect("attempt stop and restart"); + // One Escape closes the list only if neither key staged an approval or a + // lifecycle console. In either modal, it dismisses that modal instead and + // leaves the Services overlay open, which the next step then catches. + tui.send("\u{1b}").expect("close Services list"); +} + +#[then("the Services overlay closes without an approval prompt or job")] +async fn external_lifecycle_keys_left_no_modal(world: &mut E2eWorld) { + session(world) + .wait_until_gone(SERVICES_OVERLAY_TITLE, default_timeout()) + .await + .expect("external lifecycle keys must not stage approval or launch a job"); +} + +#[then("both loaded models are still displayed")] +async fn external_models_still_visible(world: &mut E2eWorld) { + external_models_visible(world).await; +} + #[then("the overlay reports the record that is no longer running")] async fn services_overlay_reports_past_attempts(world: &mut E2eWorld) { // The overlay renders only the live instances the daemon scrapes, so the diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index 180a539cea..0e7a9816fe 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -592,6 +592,119 @@ async fn assert_lemonade_backend_alignment_reported(world: &mut E2eWorld) { ); } +/// The Lemonade install `rocm serve` consults before it serves a GGUF directly: +/// the selected backend and where its `llama-server` lives. +pub(crate) struct SelectedLemonadeBackend { + manifest: serde_json::Value, + backend: String, + runtime_dir: std::path::PathBuf, +} + +impl SelectedLemonadeBackend { + /// Read the manifest the install wrote: the runtime directory and backend are + /// what the engine's own lookup (`/bin/llamacpp//`) uses. + pub(crate) fn read(world: &E2eWorld) -> Self { + let root = world + .isolated_root + .as_ref() + .expect("scenario has no isolated state root") + .path(); + let manifest_path = root.join("data/engines/lemonade/manifests/runtime.json"); + let manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(&manifest_path).unwrap_or_else(|error| { + panic!("failed to read {}: {error}", manifest_path.display()) + })) + .unwrap_or_else(|error| panic!("failed to parse {}: {error}", manifest_path.display())); + let field = |name: &str| { + manifest[name] + .as_str() + .unwrap_or_else(|| panic!("no `{name}` in {}: {manifest}", manifest_path.display())) + .to_owned() + }; + let (recipe, backend, runtime_dir) = ( + field("backend_recipe"), + field("backend_name"), + std::path::PathBuf::from(field("runtime_dir")), + ); + assert_eq!( + recipe, "llamacpp", + "the selected recipe is not llama.cpp, so direct serving has no llama-server to find: {manifest}" + ); + Self { + manifest, + backend, + runtime_dir, + } + } + + /// The backend Lemonade selected (`rocm` or `vulkan`), as the manifest records it. + pub(crate) fn backend(&self) -> &str { + &self.backend + } + + /// The selected backend's `llama-server`, or the directories searched. + fn server(&self) -> Result { + let lookup = + e2e_cucumber::lemonade_backend::find_llama_server(&self.runtime_dir, &self.backend) + .unwrap_or_else(|reason| panic!("{reason}: {}", self.manifest)); + lookup.found.ok_or_else(|| { + format!( + "no llama-server exists for the selected backend `{}` under {}. Searched (flat, or one build-numbered level down):{}", + self.backend, + self.runtime_dir.display(), + lookup + .searched + .iter() + .fold(String::new(), |mut listing, dir| { + use std::fmt::Write as _; + let _ = write!(listing, "\n {}", dir.display()); + listing + }) + ) + }) + } +} + +/// A precondition for the reinstall advice: `rocm serve` tells a user whose +/// selected backend has no `llama-server` to run +/// `rocm engines install lemonade --reinstall`. That advice only holds if the +/// command brings the file back, so the scenario first installs Lemonade (the +/// plain install, so the later `--reinstall` is what has to repair it). +#[given("the lemonade engine is installed")] +async fn setup_lemonade_engine_installed(world: &mut E2eWorld) { + crate::run_rocm_ok(world, &["engines", "install", "lemonade"]); +} + +/// Damage the install the way the advice is for: delete the selected backend's +/// `llama-server`. Asserts it was there first and is gone after, so the Then that +/// follows a reinstall can only pass if the reinstall restored it. +#[given("the selected Lemonade backend's llama-server binary has been deleted")] +async fn delete_lemonade_backend_server(world: &mut E2eWorld) { + let selected = SelectedLemonadeBackend::read(world); + let binary = selected + .server() + .unwrap_or_else(|reason| panic!("nothing to delete after the install: {reason}")); + std::fs::remove_file(&binary) + .unwrap_or_else(|error| panic!("failed to delete {}: {error}", binary.display())); + assert!( + selected.server().is_err(), + "a llama-server still exists for `{}` after deleting {}", + selected.backend, + binary.display() + ); +} + +/// Checks the file the direct serve route looks for exists. In runtime-16 it runs +/// after the file was deleted, so it is the reinstall that put it back; the +/// lookup is [`e2e_cucumber::lemonade_backend::find_llama_server`]. +#[then("the selected Lemonade backend's llama-server binary is installed")] +async fn assert_lemonade_backend_server_installed(world: &mut E2eWorld) { + let selected = SelectedLemonadeBackend::read(world); + if let Err(searched) = selected.server() { + panic!("after the reinstall, {searched}"); + } +} + /// The CLI names the variable whenever it is set, so a user can tell it was /// read -- this alone does not prove alignment was actually skipped. /// diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index c45a81b7e4..024f0b5fee 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -496,13 +496,13 @@ fn run_stop_server_tool(world: &mut E2eWorld, rocmd: PathBuf) { // ── Then ─────────────────────────────────────────────────────────── -// `render_service_action_result` only prints the `service:`/`status:`/ -// `endpoint:` lines when `output.result` carries a nested `service` object -// (as `restart_server` does); `stop_server`'s result is the flat -// `service_id`/`status` shape `stop_internal_managed_service` returns, so none -// of those lines ever print for a stop today. That mismatch is a pre-existing -// gap in that renderer, so this assertion covers only what the command -// actually prints; the on-disk status below is what proves the stop itself. +// `render_service_action_result` prints the `service:` and `status:` lines for +// `stop_server`'s flat `service_id`/`status` result as well as for the nested +// `service` object `restart_server` returns (unit-tested by +// `service_stop_render_prints_the_in_process_stop_report_shape`); `endpoint:` +// still prints only for the nested shape. This step asserts the headline and the +// two lines naming the service and its status; the on-disk status below is what +// proves the stop itself. #[then("the CLI reports the service as stopped")] async fn reports_stopped(world: &mut E2eWorld) { assert_succeeded(world); @@ -511,6 +511,15 @@ async fn reports_stopped(world: &mut E2eWorld) { combined.contains("Local server stopped"), "the stop command must say it stopped the server, got:\n{combined}" ); + let has_line = |wanted: String| combined.lines().any(|line| line.trim() == wanted); + assert!( + has_line(format!("service: {SERVICE_ID}")), + "the stop report must name the service it stopped, got:\n{combined}" + ); + assert!( + has_line("status: stopped".to_owned()), + "the stop report must give the status the service ended in, got:\n{combined}" + ); } #[then("the service record on disk is marked stopped")] diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index c6788fc0d5..9f5f4a6820 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -216,7 +216,16 @@ async fn serve_and_wait( ) -> String { let timeout_secs = serve_timeout_for(world); let device_state = ensure_serve_port_free().await; - let (stdout, stderr, rc) = crate::run_rocm(world, args); + // Bounded: a `rocm serve` that never returns must fail this scenario, not + // stall the lane until the job is cancelled. The bound is the readiness + // timeout plus a margin, so a slow cold start fails through the CLI's own + // diagnostic rather than a harness kill; readiness polling keeps the plain + // timeout. + let (stdout, stderr, rc) = crate::run_rocm_bounded( + world, + args, + e2e_cucumber::serve_log::serve_cli_budget(timeout_secs), + ); if rc == 0 && model_is_ready(&MODELS_URL, Some(ready_substr), timeout_secs).await { world.endpoint = Some(SERVE_BASE_URL.to_string()); world.model_name = Some(model.to_string()); @@ -666,8 +675,11 @@ async fn serve_gpu_model(world: &mut E2eWorld) { for attempt in 1..=attempts { made = attempt; let device_state = ensure_serve_port_free().await; - let (stdout, stderr, rc) = - crate::run_rocm(world, &["serve", model, "--engine", engine, "--managed"]); + let (stdout, stderr, rc) = crate::run_rocm_bounded( + world, + &["serve", model, "--engine", engine, "--managed"], + e2e_cucumber::serve_log::serve_cli_budget(timeout_secs), + ); if rc == 0 && model_is_ready(&MODELS_URL, Some(ready_substr), timeout_secs).await { world.endpoint = Some(SERVE_BASE_URL.to_string()); world.model_name = Some(model.to_string()); @@ -796,10 +808,10 @@ async fn serving_started_no_background_helper(world: &mut E2eWorld) { #[given("a canonical Hugging Face GGUF checkpoint is being served on lemonade")] async fn setup_lemonade_hf_checkpoint_model(world: &mut E2eWorld) { - // Forces the `owner/repo:variant` direct-serve path (`serve_hf_checkpoint`, - // EAI-8026) instead of the short-recipe-name router that `setup_lemonade_model` - // exercises. Same underlying checkpoint (unsloth/Qwen3-0.6B-GGUF, Q4_0), so the - // GGUF is already warm in the HF cache when both scenarios run in one job. + // Explicit checkpoints require the selected GPU direct-server backend on + // Linux and Windows. Named entries may also serve directly on Linux, while + // more complex catalog metadata stays with the router. Both scenarios use + // unsloth/Qwen3-0.6B-GGUF:Q4_0, sharing the same HF cache. // // This is the path #260 reports: on Strix Halo Windows the launch exits 0 and // the endpoint never answers. `serve_and_wait` is what makes that outcome @@ -934,6 +946,228 @@ async fn reserve_aim_model(world: &mut E2eWorld) { serve_aim_canary(world).await; } +/// The unmodified v11.5.1 catalog entry carries a required mmproj file. Its +/// public HF checkpoint is unsloth/Qwen3.5-0.8B-GGUF:Qwen3.5-0.8B-UD-Q4_K_XL.gguf. +const COMPLEX_REGISTERED_MODEL: &str = "Qwen3.5-0.8B-GGUF"; + +/// Select only the real service recorded in this scenario's isolated data root. +/// Neither an endpoint found on another port nor a shared runtime grants ownership. +fn complex_model_service(world: &E2eWorld) -> rocm_core::ManagedServiceRecord { + registered_model_service(world, COMPLEX_REGISTERED_MODEL) +} + +/// The scenario-owned managed Lemonade service for `model`. +fn registered_model_service(world: &E2eWorld, model: &str) -> rocm_core::ManagedServiceRecord { + let root = world + .isolated_root + .as_ref() + .expect("no isolated scenario root"); + let services_dir = root.path().join("data/services"); + let mut records = Vec::new(); + for entry in std::fs::read_dir(&services_dir).expect("managed service directory missing") { + let path = entry.expect("unreadable service entry").path(); + if path.extension().and_then(|extension| extension.to_str()) != Some("json") { + continue; + } + let mut record: rocm_core::ManagedServiceRecord = serde_json::from_slice( + &std::fs::read(&path).expect("unreadable managed service record"), + ) + .expect("invalid managed service record"); + record.normalize_paths_for_host(); + if record.engine == "lemonade" + && record.model_ref == model + && Some(record.endpoint_url.as_str()) == world.endpoint.as_deref() + { + records.push(record); + } + } + assert_eq!( + records.len(), + 1, + "expected exactly one scenario-owned {model} service" + ); + records.pop().expect("record count checked") +} + +#[given("a registered vision GGUF is being served on lemonade on its own port")] +async fn setup_complex_registered_lemonade_model(world: &mut E2eWorld) { + serve_registered_on_its_own_port(world, COMPLEX_REGISTERED_MODEL, "Qwen3.5-0.8B").await; +} + +/// A simple registered entry: one checkpoint with a variant and no extra files, +/// so on Linux Lemonade's catalog resolves it to the direct llama-server route. +const SIMPLE_REGISTERED_MODEL: &str = "Qwen3-0.6B-GGUF"; + +#[given("a simple registered GGUF is being served on lemonade on its own port")] +async fn setup_simple_registered_lemonade_model(world: &mut E2eWorld) { + serve_registered_on_its_own_port(world, SIMPLE_REGISTERED_MODEL, "Qwen3-0.6B").await; +} + +async fn serve_registered_on_its_own_port(world: &mut E2eWorld, model: &str, ready_fragment: &str) { + // Reserve an unused loopback port, then release it immediately before the + // ordinary CLI launch. A bind race fails the scenario; never stop a listener + // that happened to claim it, and never invoke the shared-port kill helper. + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("cannot reserve serving port"); + let port = listener.local_addr().expect("no reserved address").port(); + world.endpoint = Some(format!("http://127.0.0.1:{port}/v1")); + world.model_name = Some(model.to_owned()); + let port_arg = port.to_string(); + drop(listener); + let (stdout, stderr, rc) = crate::run_rocm_bounded( + world, + &[ + "serve", + model, + "--engine", + "lemonade", + "--managed", + "--host", + "127.0.0.1", + "--port", + &port_arg, + ], + e2e_cucumber::serve_log::serve_cli_budget(serve_timeout_for(world)), + ); + let models_url = format!( + "{}/models", + world.endpoint.as_deref().expect("endpoint set") + ); + let ready = rc == 0 + && model_is_ready(&models_url, Some(ready_fragment), serve_timeout_for(world)).await; + assert!( + ready, + "{}", + serve_failure_evidence( + world, + Some("lemonade"), + &format!("registered serve of {model} failed (rc={rc}, endpoint={models_url})"), + "scenario-owned port; no shared listener was stopped", + &stdout, + &stderr, + ) + .await + ); + // Reject readiness from an unrelated listener even if it serves the same model. + let record = registered_model_service(world, model); + assert_eq!(record.port, port); + assert_eq!(record.host, "127.0.0.1"); +} + +/// The service's engine state once it says `ready`, polled for up to ten seconds +/// (the state is written by the serve process, not by the CLI that returned). +async fn wait_for_ready_engine_state( + record: &rocm_core::ManagedServiceRecord, +) -> serde_json::Value { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let state = std::fs::read(&record.engine_state_path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + if let Some(state) = state + && state["status"] == "ready" + { + assert_eq!(state["engine"], "lemonade", "{state}"); + return state; + } + assert!( + Instant::now() < deadline, + "ready state missing at {}", + record.engine_state_path.display() + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +#[then("the managed engine state confirms the Lemonade router loaded the registered model")] +async fn assert_complex_model_router(world: &mut E2eWorld) { + let state = wait_for_ready_engine_state(&complex_model_service(world)).await; + assert_eq!(state["load_response"]["method"], "lemonade-cli", "{state}"); + assert_eq!( + state["load_response"]["model_name"], COMPLEX_REGISTERED_MODEL, + "{state}" + ); +} + +#[then("the managed engine state records direct serving by the selected GPU backend")] +async fn assert_simple_model_served_directly(world: &mut E2eWorld) { + let state = + wait_for_ready_engine_state(®istered_model_service(world, SIMPLE_REGISTERED_MODEL)) + .await; + // The route, not just a working endpoint: a router serve records + // `lemonade-cli`, the direct server this method. + assert_eq!( + state["load_response"]["method"], "lemonade-packaged-llama-server", + "{state}" + ); + // Which llama-server served. The direct route only ever runs one from a GPU + // backend directory, so "not CPU" cannot fail here; what can go wrong is a + // crossover, a stale directory of the other backend serving in place of the + // one Lemonade selected. The recorded directory must belong to the selected + // backend. + let selected = crate::e2e::runtime_steps::SelectedLemonadeBackend::read(world); + let dirs = + e2e_cucumber::lemonade_backend::backend_dirs(selected.backend()).unwrap_or_else(|| { + panic!( + "the selected backend `{}` has no GPU llama-server: {state}", + selected.backend() + ) + }); + let served = state["backend_requested"].as_str().unwrap_or_default(); + assert!( + dirs.contains(&served), + "served by `{served}`, which is not a directory of the selected backend `{}` ({dirs:?}): {state}", + selected.backend() + ); +} + +/// The managed Lemonade service of whichever registered model this scenario +/// served (`world.model_name`, set by the serve step), so the stop steps serve +/// both the router and the direct-route scenarios. +fn scenario_registered_service(world: &E2eWorld) -> rocm_core::ManagedServiceRecord { + let model = world + .model_name + .as_deref() + .expect("the scenario has not served a registered model"); + registered_model_service(world, model) +} + +#[when("the user stops the scenario's managed Lemonade service")] +async fn stop_complex_model_service(world: &mut E2eWorld) { + let record = scenario_registered_service(world); + let (stdout, stderr, rc) = + crate::run_rocm(world, &["services", "stop", &record.service_id, "--yes"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the managed service is stopped and its endpoint is closed")] +async fn assert_complex_model_stopped(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "stop failed: {:?}", world.cli_stderr); + let record = scenario_registered_service(world); + let output = world.cli_output.as_deref().expect("no stop command output"); + assert!(output.contains("stopped"), "{output}"); + assert!(output.contains(&record.service_id), "{output}"); + assert_eq!(record.status, "stopped"); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let connection = tokio::time::timeout( + Duration::from_secs(1), + tokio::net::TcpStream::connect(("127.0.0.1", record.port)), + ) + .await; + if matches!(connection, Ok(Err(_))) { + return; + } + assert!( + Instant::now() < deadline, + "stopped service still listens on {}", + record.endpoint_url + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + #[given("a large model is being served on GPU")] async fn setup_large_gpu_model(world: &mut E2eWorld) { // Large-model nightly coverage follows the host's serving path. vLLM hosts @@ -1092,7 +1326,11 @@ async fn user_serves_default_engine(world: &mut E2eWorld) { // fn's doc comment. let model = default_engine_serve_target(); let device_state = ensure_serve_port_free().await; - let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", model, "--managed"]); + let (stdout, stderr, rc) = crate::run_rocm_bounded( + world, + &["serve", model, "--managed"], + e2e_cucumber::serve_log::serve_cli_budget(serve_timeout_for(world)), + ); // The model the CLI resolved (what actually gets served) can differ from the // requested id, so downstream reachability/readiness checks must look for the // resolved model on the shared port; fall back to the requested id. @@ -1149,8 +1387,11 @@ async fn user_serves_vllm_capable_default(world: &mut E2eWorld) { // platform. Qwen2.5-0.5B is the smallest vLLM-preferred catalog entry. Omit // `--engine` so the CLI's own default selection is what's exercised. ensure_serve_port_free().await; - let (stdout, stderr, rc) = - crate::run_rocm(world, &["serve", "Qwen/Qwen2.5-0.5B-Instruct", "--managed"]); + let (stdout, stderr, rc) = crate::run_rocm_bounded( + world, + &["serve", "Qwen/Qwen2.5-0.5B-Instruct", "--managed"], + e2e_cucumber::serve_log::serve_cli_budget(serve_timeout_for(world)), + ); world.cli_output = Some(stdout); // See the note in user_serves_default_engine: the rc is asserted later, so // stderr must be carried along to explain a non-zero one.