diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index 114cad975e..3e182c673b 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -25,6 +25,7 @@ //! above. use crate::automations::{render_automations_text, render_daemon_text}; +use crate::engine_runtime_select::{apply_app_path_env, service_model_names_match}; use crate::freeform_planner::{ format_structured_tool_call, infer_engine_from_request, plan_freeform_request_with_context, serve_args_request_cpu_device, @@ -37,13 +38,12 @@ use crate::service_launch::is_loopback_host; use crate::uninstall_plan::render_uninstall_dry_run; use crate::update_cmd::render_update_text; use crate::{ - ChatInferenceParams, apply_app_path_env, build_codex_bridge_gpu_snapshot, - build_codex_bridge_snapshot, builtin_codex_bridge_engine_inventory, cli_version_string, - read_tail_lines, record_cli_audit_event, render_config_text, - render_engine_inventory_text_with_paths, render_examine_text, render_examine_text_with_paths, - render_logs_browser_text, render_logs_text, - render_model_registry_verbose_text_with_context_and_host, render_service_logs_text, - render_services_text, render_setup_status_text, service_model_names_match, + ChatInferenceParams, build_codex_bridge_gpu_snapshot, build_codex_bridge_snapshot, + builtin_codex_bridge_engine_inventory, cli_version_string, read_tail_lines, + record_cli_audit_event, render_config_text, render_engine_inventory_text_with_paths, + render_examine_text, render_examine_text_with_paths, render_logs_browser_text, + render_logs_text, render_model_registry_verbose_text_with_context_and_host, + render_service_logs_text, render_services_text, render_setup_status_text, therock_install_version_selector, validate_service_id, }; use crate::{chat_host_facts, comfyui, providers, therock}; diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 81434b5fec..758e5e88e3 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -46,13 +46,15 @@ //! `decode_engine_response`, `with_scoped_builtin_engine_env`, //! `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this //! module and stay private. Shared helpers this module calls back into at the -//! crate root — `apply_app_path_env`, `app_path_env_vars`, -//! `with_sigpipe_ignored`, `optional_arg`, `engine_recipe_json_arg`, -//! `device_policy_name`, `missing_packaged_engine_reason` — stay in -//! `main.rs` and are reached unqualified via `use crate::{...}`, same as for -//! other mechanically relocated modules. `parse_device_policy` moved out of -//! `main.rs` into `gpu_policy.rs` in ROCMAI-214 (Phase 6f); it is reached via -//! `use crate::gpu_policy::parse_device_policy;` instead. +//! crate root — `with_sigpipe_ignored`, `device_policy_name`, +//! `missing_packaged_engine_reason` — stay in `main.rs` and are reached +//! unqualified via `use crate::{...}`, same as for other mechanically +//! relocated modules. `parse_device_policy` moved out of `main.rs` into +//! `gpu_policy.rs` in ROCMAI-214 (Phase 6f); it is reached via `use +//! crate::gpu_policy::parse_device_policy;` instead. `apply_app_path_env`, +//! `app_path_env_vars`, `optional_arg`, and `engine_recipe_json_arg` moved out +//! of `main.rs` into `engine_runtime_select.rs` in ROCMAI-215 (Phase 6f); this +//! file's `use` list below points there now instead of at the crate root. use std::ffi::OsString; use std::io::{self, BufRead, Read, Write}; @@ -70,11 +72,11 @@ use rocm_engine_protocol::{ use serde::Serialize; use serde::de::DeserializeOwned; -use crate::gpu_policy::parse_device_policy; -use crate::{ - app_path_env_vars, apply_app_path_env, device_policy_name, engine_recipe_json_arg, - missing_packaged_engine_reason, optional_arg, with_sigpipe_ignored, +use crate::engine_runtime_select::{ + app_path_env_vars, apply_app_path_env, engine_recipe_json_arg, optional_arg, }; +use crate::gpu_policy::parse_device_policy; +use crate::{device_policy_name, missing_packaged_engine_reason, with_sigpipe_ignored}; static BUILTIN_ENGINE_ENV_LOCK: OnceLock> = OnceLock::new(); diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs new file mode 100644 index 0000000000..7ba7a50d81 --- /dev/null +++ b/apps/rocm/src/engine_runtime_select.rs @@ -0,0 +1,1036 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Engine/runtime selection and HTTP readiness polling: deciding which +//! runtime (or TheRock env) a `serve`/`engines install` invocation resolves +//! to (`EngineSelection`, `resolve_engine_selection`, +//! `validate_engine_selection_runtime`, `single_ready_runtime_key`, +//! `resolve_runtime_selector_to_exact_key`), small argv-building helpers used +//! when relaunching a managed engine (`optional_arg`, +//! `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, +//! `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` +//! environment variables every managed child inherits +//! (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/ +//! `apply_app_path_env`), and polling a freshly launched service until it can +//! actually answer inference requests (`wait_for_service_http_ready`, +//! `wait_for_service_http_ready_with_progress`, +//! `service_http_readiness_paths`, `http_get_local_service`, +//! `http_post_local_service_json`, `service_http_readiness_response_ready`, +//! `lemonade_health_ready_for_model`, `model_list_ready_for_model`, +//! `service_model_gpu_ready`, `service_model_reports_rocm_backend`, +//! `service_model_names_match`). +//! +//! Mechanically relocated from `main.rs` with no behavior change — Phase 6f +//! (ROCMAI-215) of the `main.rs` modularization (see +//! `docs/architecture.md`). This is the last sub-phase of the `main.rs` +//! Phase 6f split that lands before the file's single shared +//! `#[cfg(test)] mod tests` block starts: the function immediately after this +//! cluster in the original file, `treat_as_natural_language`, is +//! dispatch-adjacent natural-language-invocation routing left over from an +//! earlier phase (ROCMAI-87) and stays in `main.rs` on purpose — it is not +//! part of this module and is not touched by this change. +//! +//! One item physically interleaved in this span in `main.rs` stayed there +//! instead of moving: `const fn status_for_readiness` sits between +//! `wait_for_service_http_ready_with_progress` and +//! `service_http_readiness_paths`, but it already had callers outside this +//! cluster — `main.rs`'s own `restart_internal_managed_service_with` and +//! `service_launch.rs`, which reaches it unqualified via `use +//! crate::status_for_readiness;` — so it stays a shared crate-root +//! function, reached from here (and from the retained tests below) the same +//! way, via `use crate::status_for_readiness;`. +//! +//! `EngineSelection` is not named in isolation anywhere in `main.rs`'s +//! dispatch/call-site re-import contract, but it is this cluster's own type — +//! returned by `resolve_engine_selection`/`validate_engine_selection_runtime` +//! and read nowhere else — so it moves with the functions that produce it, +//! the same way `serve_cmd.rs`'s `ServeEngineSelection` stayed with +//! `select_serve_engine`. +//! +//! `resolve_engine_selection` and `validate_engine_selection_runtime` are +//! `pub(crate)`, read from `engines_cmd.rs` and `serve_cmd.rs` (neither is +//! called from `main.rs` itself outside its own now-relocated tests); +//! `EngineSelection`, and its fields, are `pub(crate)` too, since those two +//! callers read `.runtime_id`/`.env_id`/`.source` directly off the value +//! `resolve_engine_selection`/`validate_engine_selection_runtime` return. +//! `resolve_runtime_selector_to_exact_key` is also `pub(crate)`: `engines_cmd.rs` +//! calls it in production, and `runtimes_cmd.rs`'s own test module calls it +//! directly too. `single_ready_runtime_key` is called only from +//! `validate_engine_selection_runtime` in this same module and stays private. +//! `optional_arg` and +//! `engine_recipe_json_arg` are `pub(crate)`, read from `engine_ipc.rs`'s +//! `engine_request`; `parse_engine_recipe_json_arg` is `pub(crate)`, read +//! from `main.rs`'s `dispatch()` and `restart_child_argv`. +//! `app_path_env_vars`/`apply_app_path_env` are `pub(crate)`, read from +//! `engine_ipc.rs`, and `apply_app_path_env` is also read from `chat.rs`, +//! `engines_cmd.rs`, `service_launch.rs`, and `main.rs`'s +//! `restart_internal_managed_service_with`; `app_path_env_var_values`/ +//! `app_path_env_var_refs` are `pub(crate)`, read from `service_launch.rs` +//! and `main.rs`'s `restart_internal_managed_service_with`. +//! `daemon_run_argv` is `pub(crate)`, read from `main.rs`'s `dispatch()`; +//! `managed_service_launcher_path` is `pub(crate)`, read from +//! `service_launch.rs` and `main.rs`'s `restart_internal_managed_service_with`. +//! `wait_for_service_http_ready` is `pub(crate)`, read from `main.rs`'s +//! `restart_internal_managed_service_with`; `wait_for_service_http_ready_with_progress` +//! is `pub(crate)`, read from `service_launch.rs`. `http_post_local_service_json` +//! is `pub(crate)`, read from `main.rs`'s `unload_lemonade_service_model`. +//! `service_model_names_match` is `pub(crate)`, read from `chat.rs`. +//! `service_http_readiness_paths`, `http_get_local_service`, +//! `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, +//! `model_list_ready_for_model`, `service_model_gpu_ready`, and +//! `service_model_reports_rocm_backend` have no callers outside this module +//! and stay private. +//! +//! The cluster's regression tests live in this module's own `mod tests`, +//! reaching the shared `test_app_paths` fixture (needed by the +//! `app_path_env_*` tests) through `crate::tests::test_app_paths` the same +//! way other mechanically relocated modules reach shared fixtures, and +//! `status_for_readiness` (needed by the readiness-wait tests) through +//! `crate::status_for_readiness`, since it stays at the crate root per above. + +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::process::Command as ProcessCommand; +use std::thread; +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use rocm_core::{ + AppPaths, EndpointReadiness, RocmCliConfig, connect_tcp_stream, daemon_binary_path, + format_host_port, format_http_base_url, read_http_response_bounded, write_all_tcp_stream, +}; +use rocm_engine_protocol::EngineRecipeHint; + +use crate::runtimes_cmd::{ + recover_setup_runtime_registration, select_runtime_manifest, + validate_runtime_manifest_for_activation, +}; +use crate::therock; + +#[derive(Debug, Clone)] +pub(crate) struct EngineSelection { + pub(crate) runtime_id: Option, + pub(crate) env_id: Option, + pub(crate) source: Option, +} + +pub(crate) fn resolve_engine_selection( + config: &RocmCliConfig, + engine: &str, + runtime_id: Option<&str>, + env_id: Option<&str>, +) -> EngineSelection { + if let Some(env_id) = env_id { + return EngineSelection { + runtime_id: None, + env_id: Some(env_id.to_owned()), + source: Some("cli_env_id".to_owned()), + }; + } + if let Some(runtime_id) = runtime_id { + return EngineSelection { + runtime_id: Some(runtime_id.to_owned()), + env_id: None, + source: Some("cli_runtime_id".to_owned()), + }; + } + + if let Some(runtime_key) = config.active_runtime_key.as_ref() { + return EngineSelection { + runtime_id: Some(runtime_key.clone()), + env_id: None, + source: Some("config_active_runtime_key".to_owned()), + }; + } + + if let Some(entry) = config.engine_config(engine) { + if let Some(env_id) = entry.preferred_env_id.as_ref() { + return EngineSelection { + runtime_id: None, + env_id: Some(env_id.clone()), + source: Some("config_preferred_env_id".to_owned()), + }; + } + if let Some(runtime_id) = entry.preferred_runtime_id.as_ref() { + return EngineSelection { + runtime_id: Some(runtime_id.clone()), + env_id: None, + source: Some("config_preferred_runtime_id".to_owned()), + }; + } + if let Some(env_id) = entry.last_installed_env_id.as_ref() { + return EngineSelection { + runtime_id: None, + env_id: Some(env_id.clone()), + source: Some("config_last_installed_env_id".to_owned()), + }; + } + if let Some(runtime_id) = entry.last_installed_runtime_id.as_ref() { + return EngineSelection { + runtime_id: Some(runtime_id.clone()), + env_id: None, + source: Some("config_last_installed_runtime_id".to_owned()), + }; + } + } + + if let Some(runtime_id) = config.default_runtime_id.as_ref() { + return EngineSelection { + runtime_id: Some(runtime_id.clone()), + env_id: None, + source: Some("config_default_runtime_id".to_owned()), + }; + } + + EngineSelection { + runtime_id: None, + env_id: None, + source: None, + } +} + +pub(crate) fn validate_engine_selection_runtime( + paths: &AppPaths, + mut selection: EngineSelection, +) -> Result { + if let Some(runtime_id) = selection.runtime_id.as_deref() { + let source = selection.source.as_deref().unwrap_or("runtime selection"); + selection.runtime_id = Some(resolve_runtime_selector_to_exact_key( + paths, runtime_id, source, + )?); + } else if selection.env_id.is_none() + && let Some(runtime_key) = single_ready_runtime_key(paths)? + { + selection.runtime_id = Some(runtime_key); + selection.source = Some("single_ready_runtime".to_owned()); + } + Ok(selection) +} + +fn single_ready_runtime_key(paths: &AppPaths) -> Result> { + let config = RocmCliConfig::load(paths).unwrap_or_default(); + recover_setup_runtime_registration(paths, &config)?; + let manifests = therock::load_runtime_manifests(paths)?; + let ready = manifests + .iter() + .filter(|manifest| validate_runtime_manifest_for_activation(manifest).is_ok()) + .collect::>(); + Ok(match ready.as_slice() { + [manifest] => Some(manifest.runtime_key.clone()), + _ => None, + }) +} + +pub(crate) fn resolve_runtime_selector_to_exact_key( + paths: &AppPaths, + selector: &str, + source: &str, +) -> Result { + let manifests = therock::load_runtime_manifests(paths)?; + match select_runtime_manifest(&manifests, selector) { + Ok(manifest) => Ok(manifest.runtime_key.clone()), + Err(error) => { + let config = RocmCliConfig::load(paths).unwrap_or_default(); + if recover_setup_runtime_registration(paths, &config)?.is_some() { + let manifests = therock::load_runtime_manifests(paths)?; + if let Ok(manifest) = select_runtime_manifest(&manifests, selector) { + return Ok(manifest.runtime_key.clone()); + } + } + bail!( + "runtime selector `{selector}` from {source} is not an exact usable runtime: {error}; run `rocm runtimes list` and `rocm runtimes activate `, or pass --runtime-id " + ) + } + } +} + +pub(crate) fn optional_arg(flag: &str, value: Option<&str>) -> Vec { + match value { + Some(value) => vec![flag.to_owned(), value.to_owned()], + None => Vec::new(), + } +} + +pub(crate) fn engine_recipe_json_arg( + engine_recipe: Option<&EngineRecipeHint>, +) -> Result> { + match engine_recipe { + Some(engine_recipe) => Ok(vec![ + "--engine-recipe-json".to_owned(), + serde_json::to_string(engine_recipe).context("failed to encode engine recipe hint")?, + ]), + None => Ok(Vec::new()), + } +} + +pub(crate) fn parse_engine_recipe_json_arg( + value: Option, +) -> Result> { + value + .map(|value| { + serde_json::from_str(&value).context("failed to parse --engine-recipe-json payload") + }) + .transpose() +} + +pub(crate) fn app_path_env_vars(paths: &AppPaths) -> [(&'static str, &Path); 3] { + [ + ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), + ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), + ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), + ] +} + +#[cfg_attr(not(windows), allow(dead_code))] +pub(crate) fn app_path_env_var_values( + paths: &AppPaths, + env_root: Option<&Path>, +) -> Vec<(&'static str, PathBuf)> { + let mut vars = app_path_env_vars(paths) + .into_iter() + .map(|(key, value)| (key, value.to_path_buf())) + .collect::>(); + if let Some(env_root) = env_root { + vars.push(("ROCM_CLI_ENGINE_ENVS_ROOT", env_root.to_path_buf())); + } + vars +} + +#[cfg_attr(not(windows), allow(dead_code))] +pub(crate) fn app_path_env_var_refs<'a>( + vars: &'a [(&'static str, PathBuf)], +) -> Vec<(&'static str, &'a Path)> { + vars.iter() + .map(|(key, value)| (*key, value.as_path())) + .collect() +} + +/// Argv passed to the embedded `rocmd` library to run the real foreground +/// automation loop (the same path as `rocmd run --automations-enabled`). +pub(crate) fn daemon_run_argv() -> Vec { + vec![ + OsString::from("rocmd"), + OsString::from("run"), + OsString::from("--automations-enabled"), + ] +} + +pub(crate) fn managed_service_launcher_path() -> Result { + let current_exe = daemon_binary_path()?; + if rocm_core::runtime_is_windows() { + return Ok(rocm_core::normalize_runtime_path_for_storage(¤t_exe)); + } + Ok(current_exe) +} + +pub(crate) fn apply_app_path_env(command: &mut ProcessCommand, paths: &AppPaths) { + for (key, value) in app_path_env_vars(paths) { + command.env(key, value); + } +} + +pub(crate) fn wait_for_service_http_ready( + engine: &str, + host: &str, + port: u16, + canonical_model_id: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> EndpointReadiness { + wait_for_service_http_ready_with_progress( + engine, + host, + port, + &|| canonical_model_id.to_owned(), + endpoint_api_key, + timeout, + &mut |_elapsed| {}, + ) +} + +/// Wait until the service can actually serve, reporting how far it got. +/// +/// A health or model-listing endpoint answering is not enough to call a service +/// ready: engines advertise a model within seconds of accepting its name, while +/// the weights can take minutes to become usable, so a caller that sends its +/// first request on that signal gets a hang. Only [`EndpointReadiness::Serving`] +/// — an inference request that came back — means ready; a service that lists the +/// model without answering one is [`EndpointReadiness::Listing`], and the best +/// result seen before `timeout` is what gets returned. +/// +/// Polls until `timeout` elapses, invoking `on_tick(elapsed)` once per iteration +/// so a caller can animate a spinner. Engine-neutral: `service_http_readiness_paths` +/// maps each engine to the right listing path. `endpoint_api_key` is sent as a +/// bearer token so both the listing check and the inference probe still succeed +/// against a public endpoint that requires authentication. +pub(crate) fn wait_for_service_http_ready_with_progress( + engine: &str, + host: &str, + port: u16, + model_name: &dyn Fn() -> String, + endpoint_api_key: Option<&str>, + timeout: Duration, + on_tick: &mut dyn FnMut(Duration), +) -> EndpointReadiness { + let start = std::time::Instant::now(); + let endpoint = format_http_base_url(host, port); + let mut best = EndpointReadiness::Unreachable; + while start.elapsed() < timeout { + // Re-evaluated each pass: a container engine reports the name its server + // answers to only after it starts, so a name fixed up front can be wrong. + let canonical_model_id = model_name(); + let canonical_model_id = canonical_model_id.as_str(); + let listed = service_http_readiness_paths(engine).iter().any(|path| { + http_get_local_service( + host, + port, + path, + endpoint_api_key, + Duration::from_millis(750), + ) + .is_ok_and(|(status, body)| { + service_http_readiness_response_ready( + engine, + path, + status, + &body, + canonical_model_id, + ) + }) + }); + if listed { + best = EndpointReadiness::Listing; + if rocm_core::openai_chat_completion_probe( + &endpoint, + canonical_model_id, + endpoint_api_key, + rocm_core::INFERENCE_PROBE_TIMEOUT, + ) + .unwrap_or(false) + { + return EndpointReadiness::Serving; + } + } + on_tick(start.elapsed()); + thread::sleep(Duration::from_millis(250)); + } + best +} + +fn service_http_readiness_paths(engine: &str) -> &'static [&'static str] { + match engine { + "lemonade" => &["/v1/health", "/v1/models"], + _ => &["/v1/models", "/v1/health", "/health", "/healthz"], + } +} + +fn http_get_local_service( + host: &str, + port: u16, + path: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result<(u16, String)> { + let deadline = std::time::Instant::now() + timeout; + let mut stream = connect_tcp_stream(host, port, timeout)?; + let host_header = format_host_port(host, port); + // Authenticate the probe when the endpoint is protected; loopback endpoints + // pass `None` and the header is omitted. + let auth_header = match endpoint_api_key { + Some(key) => format!("Authorization: Bearer {key}\r\n"), + None => String::new(), + }; + let request = format!( + "GET {path} HTTP/1.1\r\nHost: {host_header}\r\n{auth_header}Connection: close\r\n\r\n" + ); + write_all_tcp_stream(&mut stream, request.as_bytes()) + .context("failed to write service readiness request")?; + let response = read_http_response_bounded(&mut stream, deadline) + .context("failed to read service readiness response")?; + let (headers, body) = response + .split_once("\r\n\r\n") + .unwrap_or((response.as_str(), "")); + let status = headers + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + Ok((status, body.to_owned())) +} + +pub(crate) fn http_post_local_service_json( + host: &str, + port: u16, + path: &str, + body: &serde_json::Value, + timeout: Duration, +) -> Result<(u16, String)> { + let deadline = std::time::Instant::now() + timeout; + let mut stream = connect_tcp_stream(host, port, timeout)?; + let host_header = format_host_port(host, port); + let body = serde_json::to_string(body).context("failed to serialize service request")?; + let request = format!( + "POST {path} HTTP/1.1\r\nHost: {host_header}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + write_all_tcp_stream(&mut stream, request.as_bytes()) + .context("failed to write service request")?; + let response = read_http_response_bounded(&mut stream, deadline) + .context("failed to read service response")?; + let (headers, body) = response + .split_once("\r\n\r\n") + .unwrap_or((response.as_str(), "")); + let status = headers + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + Ok((status, body.to_owned())) +} + +fn service_http_readiness_response_ready( + engine: &str, + path: &str, + status: u16, + body: &str, + canonical_model_id: &str, +) -> bool { + if status != 200 { + return false; + } + match (engine, path) { + ("lemonade", "/v1/health") => lemonade_health_ready_for_model(body, canonical_model_id), + ("lemonade", "/v1/models") => model_list_ready_for_model(body, canonical_model_id, true), + (_, "/v1/models") => model_list_ready_for_model(body, canonical_model_id, false), + _ => false, + } +} + +fn lemonade_health_ready_for_model(body: &str, canonical_model_id: &str) -> bool { + let Ok(value) = serde_json::from_str::(body.trim()) else { + return false; + }; + value + .get("all_models_loaded") + .and_then(serde_json::Value::as_array) + .is_some_and(|models| { + models.iter().any(|model| { + let name_matches = ["model_name", "id", "name"] + .into_iter() + .filter_map(|field| model.get(field).and_then(serde_json::Value::as_str)) + .any(|loaded| service_model_names_match(loaded, canonical_model_id)); + name_matches && service_model_reports_rocm_backend(model) + }) + }) +} + +fn model_list_ready_for_model( + body: &str, + canonical_model_id: &str, + require_rocm_backend: bool, +) -> bool { + let Ok(value) = serde_json::from_str::(body.trim()) else { + return false; + }; + value + .get("data") + .and_then(serde_json::Value::as_array) + .is_some_and(|models| { + models.iter().any(|model| { + let name_matches = ["id", "model", "name"] + .into_iter() + .filter_map(|field| model.get(field).and_then(serde_json::Value::as_str)) + .any(|loaded| service_model_names_match(loaded, canonical_model_id)); + name_matches && (!require_rocm_backend || service_model_gpu_ready(model)) + }) + }) +} + +/// Whether a lemonade `/v1/models` entry is served on GPU. A stock `llama-server` +/// (direct-serve) entry has no `recipe_options` and is accepted (that path only runs +/// GPU backends); a Lemonade-router entry carries `recipe_options`, so it must report a +/// ROCm backend — which keeps a registered-but-unloaded model (empty `recipe_options`) +/// from reading as ready. +fn service_model_gpu_ready(model: &serde_json::Value) -> bool { + match model.get("recipe_options") { + None => true, + Some(_) => service_model_reports_rocm_backend(model), + } +} + +fn service_model_reports_rocm_backend(model: &serde_json::Value) -> bool { + model + .get("recipe_options") + .and_then(|options| options.get("llamacpp_backend")) + .or_else(|| model.get("llamacpp_backend")) + .and_then(serde_json::Value::as_str) + .is_some_and(|backend| backend.trim().to_ascii_lowercase().starts_with("rocm")) +} + +pub(crate) fn service_model_names_match(left: &str, right: &str) -> bool { + let left = left.trim(); + let right = right.trim(); + if left.is_empty() || right.is_empty() { + return false; + } + if left.eq_ignore_ascii_case(right) { + return true; + } + let left = left + .trim_end_matches(".gguf") + .trim_end_matches(".safetensors") + .to_ascii_lowercase(); + let right = right + .trim_end_matches(".gguf") + .trim_end_matches(".safetensors") + .to_ascii_lowercase(); + left.contains(&right) || right.contains(&left) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::status_for_readiness; + use crate::test_support::write_test_pip_runtime; + use crate::tests::{test_app_paths, test_paths}; + + #[test] + fn service_http_readiness_requires_loaded_lemonade_model() { + let loading = json!({ "all_models_loaded": [] }).to_string(); + assert!(!service_http_readiness_response_ready( + "lemonade", + "/v1/health", + 200, + &loading, + "Qwen3-0.6B-GGUF" + )); + + let loaded = json!({ + "all_models_loaded": [{ + "model_name": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": "rocm" } + }] + }) + .to_string(); + assert!(service_http_readiness_response_ready( + "lemonade", + "/v1/health", + 200, + &loaded, + "Qwen3-0.6B-GGUF" + )); + + let loaded_cpu = json!({ + "all_models_loaded": [{ + "model_name": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": "cpu" } + }] + }) + .to_string(); + assert!(!service_http_readiness_response_ready( + "lemonade", + "/v1/health", + 200, + &loaded_cpu, + "Qwen3-0.6B-GGUF" + )); + } + + #[test] + fn service_http_readiness_requires_model_list_entry() { + let empty = json!({ "data": [] }).to_string(); + assert!(!service_http_readiness_response_ready( + "vllm", + "/v1/models", + 200, + &empty, + "tiny.gguf" + )); + + let models = json!({ "data": [{ "id": "tiny.gguf" }] }).to_string(); + assert!(service_http_readiness_response_ready( + "vllm", + "/v1/models", + 200, + &models, + "tiny.gguf" + )); + + let lemonade_cpu_models = json!({ + "data": [{ + "id": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": "cpu" } + }] + }) + .to_string(); + assert!(!service_http_readiness_response_ready( + "lemonade", + "/v1/models", + 200, + &lemonade_cpu_models, + "Qwen3-0.6B-GGUF" + )); + + let lemonade_rocm_models = json!({ + "data": [{ + "id": "Qwen3-0.6B-GGUF", + "recipe_options": { "llamacpp_backend": "rocm" } + }] + }) + .to_string(); + assert!(service_http_readiness_response_ready( + "lemonade", + "/v1/models", + 200, + &lemonade_rocm_models, + "Qwen3-0.6B-GGUF" + )); + + assert!(!service_http_readiness_response_ready( + "vllm", + "/health", + 200, + "OK", + "tiny.gguf" + )); + assert!(!service_http_readiness_response_ready( + "vllm", + "/healthz", + 200, + "OK", + "Qwen3-0.6B-GGUF" + )); + } + + #[test] + fn lemonade_direct_serve_model_reads_ready_without_recipe_options() { + // The HF direct-serve path runs a stock llama-server whose `/v1/models` entry + // has no `recipe_options`. It must read as ready by name (that path is GPU-only), + // while a registered-but-unloaded lemonade entry (empty `recipe_options`) must not. + let direct = json!({ + "data": [{ "id": "LiquidAI/LFM2.5-230M-GGUF:Q4_0", "object": "model" }] + }) + .to_string(); + assert!(service_http_readiness_response_ready( + "lemonade", + "/v1/models", + 200, + &direct, + "LiquidAI/LFM2.5-230M-GGUF:Q4_0" + )); + + let registered = json!({ + "data": [{ "id": "LiquidAI/LFM2.5-230M-GGUF:Q4_0", "recipe_options": {} }] + }) + .to_string(); + assert!(!service_http_readiness_response_ready( + "lemonade", + "/v1/models", + 200, + ®istered, + "LiquidAI/LFM2.5-230M-GGUF:Q4_0" + )); + } + + #[test] + fn serve_readiness_wait_withholds_ready_while_the_model_only_lists() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + + // This is the wait behind `rocm serve`, and its verdict is what + // `services list` later prints. A model listing on `/v1/models` while + // inference still fails must come back as `Listing` — reporting it ready + // is the false positive users and automation trip over. + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + let server = thread::spawn(move || -> Result<()> { + // Serve until the wait below gives up and the test drops the socket. + while let Ok((mut stream, _)) = listener.accept() { + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 1024]; + let Ok(read) = stream.read(&mut buffer) else { + continue; + }; + let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); + let (status_line, body) = if request.starts_with("POST /v1/chat/completions ") { + ("HTTP/1.1 503 Service Unavailable", r#"{"error":"loading"}"#) + } else { + ("HTTP/1.1 200 OK", r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"#) + }; + let _ = write!( + stream, + "{status_line}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + } + Ok(()) + }); + + let readiness = wait_for_service_http_ready( + "vllm", + "127.0.0.1", + port, + "Qwen3-0.6B-GGUF", + None, + Duration::from_millis(600), + ); + + assert_eq!( + readiness, + EndpointReadiness::Listing, + "a listed-but-unservable model is not ready" + ); + assert_eq!( + status_for_readiness(readiness), + "running", + "a loading service must not be recorded as `starting`, which rocmd \ + restarts once stale" + ); + drop(server); + Ok(()) + } + + #[test] + fn serve_readiness_wait_reports_ready_once_inference_answers() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + let server = thread::spawn(move || -> Result<()> { + while let Ok((mut stream, _)) = listener.accept() { + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 1024]; + let Ok(read) = stream.read(&mut buffer) else { + continue; + }; + let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); + let body = if request.starts_with("POST /v1/chat/completions ") { + r#"{"choices":[{"message":{"content":"ok"}}]}"# + } else { + r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"# + }; + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + } + Ok(()) + }); + + let readiness = wait_for_service_http_ready( + "vllm", + "127.0.0.1", + port, + "Qwen3-0.6B-GGUF", + None, + Duration::from_secs(5), + ); + + assert_eq!(readiness, EndpointReadiness::Serving); + assert_eq!(status_for_readiness(readiness), "ready"); + drop(server); + Ok(()) + } + + #[test] + fn serve_readiness_ready_verdict_does_not_wait_for_the_peer_to_close() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::time::Instant; + + // Regression test for the other half of this PR's fix: a response is + // read to completion by its own framing, not by waiting for the peer + // to close. Before this fix, `read_tcp_stream_to_string` blocked + // until EOF, so a keep-alive engine that answers correctly but never + // closes the socket looked identical to a hung one — the readiness + // probe ran out its timeout and reported not-ready even though the + // answer had already arrived. + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + let server = thread::spawn(move || { + while let Ok((mut stream, _)) = listener.accept() { + thread::spawn(move || { + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 1024]; + let Ok(read) = stream.read(&mut buffer) else { + return; + }; + let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); + let body = if request.starts_with("POST /v1/chat/completions ") { + r#"{"choices":[{"message":{"content":"ok"}}]}"# + } else { + r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"# + }; + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", + body.len(), + body + ); + // Hold the connection open well past the readiness wait's + // timeout below, and deliberately omit `Connection: + // close`. The client must not need EOF to recognize the + // response as complete. + thread::sleep(Duration::from_secs(10)); + }); + } + }); + + let started = Instant::now(); + let readiness = wait_for_service_http_ready( + "vllm", + "127.0.0.1", + port, + "Qwen3-0.6B-GGUF", + None, + Duration::from_secs(5), + ); + + assert_eq!(readiness, EndpointReadiness::Serving); + assert!( + started.elapsed() < Duration::from_secs(3), + "a complete response must be recognized without waiting on the peer to close" + ); + // `server`'s accept loop runs forever; dropping the JoinHandle detaches + // it rather than joining, and the thread dies with the test process. + drop(server); + Ok(()) + } + + #[test] + fn app_path_env_vars_include_config_data_and_cache() { + let paths = test_app_paths(); + let vars = app_path_env_vars(&paths); + + assert_eq!(vars[0], ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path())); + assert_eq!(vars[1], ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path())); + assert_eq!(vars[2], ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path())); + } + + #[test] + fn app_path_env_var_values_include_engine_env_root_when_needed() { + let paths = test_app_paths(); + let engine_root = PathBuf::from("D:/rocm-data/runtime/engines"); + let vars = app_path_env_var_values(&paths, Some(&engine_root)); + + assert_eq!( + vars.last().map(|(key, value)| (*key, value.as_path())), + Some(("ROCM_CLI_ENGINE_ENVS_ROOT", engine_root.as_path())) + ); + } + + #[test] + fn resolve_engine_selection_uses_default_runtime_after_engine_prefs() { + let mut config = RocmCliConfig { + default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), + ..RocmCliConfig::default() + }; + + let selection = resolve_engine_selection(&config, "vllm", None, None); + assert_eq!( + selection.runtime_id.as_deref(), + Some("therock-release:gfx120X-all") + ); + assert_eq!( + selection.source.as_deref(), + Some("config_default_runtime_id") + ); + + config.active_runtime_key = Some("release-pip-gfx120x-all-7-13-0".to_owned()); + let selection = resolve_engine_selection(&config, "vllm", None, None); + assert_eq!( + selection.runtime_id.as_deref(), + Some("release-pip-gfx120x-all-7-13-0") + ); + assert_eq!( + selection.source.as_deref(), + Some("config_active_runtime_key") + ); + + config.engine_config_mut("vllm").preferred_runtime_id = + Some("therock-nightly:gfx120X-all".to_owned()); + let selection = resolve_engine_selection(&config, "vllm", None, None); + assert_eq!( + selection.runtime_id.as_deref(), + Some("release-pip-gfx120x-all-7-13-0") + ); + assert_eq!( + selection.source.as_deref(), + Some("config_active_runtime_key") + ); + + config.active_runtime_key = None; + let selection = resolve_engine_selection(&config, "vllm", None, None); + assert_eq!( + selection.runtime_id.as_deref(), + Some("therock-nightly:gfx120X-all") + ); + assert_eq!( + selection.source.as_deref(), + Some("config_preferred_runtime_id") + ); + } + + #[test] + fn engine_selection_uses_single_ready_runtime_without_active_marker() -> Result<()> { + let (root, paths) = test_paths("single-ready-runtime-selection"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + + assert_eq!( + selection.runtime_id.as_deref(), + Some(manifest.runtime_key.as_str()) + ); + assert_eq!(selection.source.as_deref(), Some("single_ready_runtime")); + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn engine_selection_keeps_multiple_ready_runtimes_explicit() -> Result<()> { + let (root, paths) = test_paths("multiple-ready-runtime-selection"); + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + + assert!(selection.runtime_id.is_none()); + assert!(selection.env_id.is_none()); + assert!(selection.source.is_none()); + let _ = std::fs::remove_dir_all(root); + Ok(()) + } +} diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index 9a25329373..ecd9da6a24 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -39,6 +39,10 @@ use serde::Deserialize; use crate::EnginesCommand; use crate::engine_ipc::{engine_request, engine_request_with_env_root}; +use crate::engine_runtime_select::{ + apply_app_path_env, resolve_engine_selection, resolve_runtime_selector_to_exact_key, + validate_engine_selection_runtime, +}; use crate::runtime_install_finalize::{ ensure_libatomic_for_torch, ensure_libnuma_for_torch, ensure_openmpi_for_vllm, settle_engine_install, @@ -48,11 +52,7 @@ use crate::runtimes_cmd::{ validate_runtime_manifest_for_activation, }; use crate::therock; -use crate::{ - apply_app_path_env, record_cli_audit_event, render_engine_inventory_text_with_paths, - resolve_engine_selection, resolve_runtime_selector_to_exact_key, - validate_engine_selection_runtime, -}; +use crate::{record_cli_audit_event, render_engine_inventory_text_with_paths}; pub(crate) fn render_engine_inventory_text() -> String { let paths = AppPaths::discover().ok(); diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 710318db9d..cc619e256b 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -24,6 +24,7 @@ mod deletion_properties; mod driver_install; mod endpoint_keys; mod engine_ipc; +mod engine_runtime_select; mod engines_cmd; mod freeform_execution; mod freeform_planner; @@ -68,12 +69,17 @@ mod update_cmd; // `runtime_install_finalize` directly (not via `dispatch()`), and several // root-level chat/examine helpers call into `runtimes_cmd` directly for the // same reason `engines_cmd`'s runtime-resolution helpers are re-imported; -// `current_runtime_manifest`/`select_runtime_manifest` stay re-imported here -// for `single_ready_runtime_key`/`resolve_runtime_selector_to_exact_key` -// below, also outside `dispatch()`. `update_cmd.rs`'s -// `apply_runtime_update`/`append_update_activate_summary` call into -// `runtimes_cmd` directly too, via their own imports rather than this -// re-export. +// `current_runtime_manifest` stays re-imported here for other root-level +// commands outside `dispatch()` that call it directly, and +// `recover_setup_runtime_registration`/`validate_runtime_manifest_for_activation` +// stay re-imported here for the same reason. `select_runtime_manifest` is no +// longer needed here: `single_ready_runtime_key`/ +// `resolve_runtime_selector_to_exact_key` moved out to +// `engine_runtime_select.rs` in ROCMAI-215 (Phase 6f), reaching it via that +// module's own grouped `use crate::runtimes_cmd::{...}` import instead. +// `update_cmd.rs`'s `apply_runtime_update`/`append_update_activate_summary` +// call into `runtimes_cmd` directly too, via their own imports rather than +// this re-export. use crate::automations::{automations, render_daemon_text}; use crate::driver_install::{install_driver, reconcile_driver_install}; use crate::engine_ipc::{ @@ -81,6 +87,14 @@ use crate::engine_ipc::{ resolve_engine_binary_path, resolve_engine_binary_path_with_paths, run_builtin_engine_serve_http, run_builtin_engine_stdio, }; +#[cfg(not(windows))] +use crate::engine_runtime_select::apply_app_path_env; +#[cfg(windows)] +use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; +use crate::engine_runtime_select::{ + daemon_run_argv, http_post_local_service_json, managed_service_launcher_path, + parse_engine_recipe_json_arg, wait_for_service_http_ready, +}; use crate::engines_cmd::{ engine_manages_own_runtime, engines, env_root_for_service, runtime_manifest_for_selector, }; @@ -102,8 +116,7 @@ use crate::runtime_install_finalize::{ use crate::runtimes_cmd::{ active_runtime_marker_path, current_runtime_manifest, default_runtime_id_matches, recover_setup_runtime_registration, runtime_keys_text, runtime_usability_status, runtimes, - select_runtime_manifest, setup_runtime_paths, toolchain_state_text, - validate_runtime_manifest_for_activation, + setup_runtime_paths, toolchain_state_text, validate_runtime_manifest_for_activation, }; use crate::serve_cmd::{ServeArgs, select_serve_engine, serve}; use crate::service_launch::ensure_public_service_has_endpoint_key; @@ -127,32 +140,27 @@ use rocm_core::{ CodexBridgeSnapshot, DEFAULT_LOCAL_HOST, EndpointReadiness, ExamineSummary, ManagedServiceRecord, ModelRecipeRecord, ModelRecipeRegistry, ModelRecipeRegistrySource, PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, RocmCliConfig, TELEMETRY_MODE_LOCAL, - TELEMETRY_MODE_OFF, WatcherMode, append_audit_event, builtin_model_recipes, connect_tcp_stream, - daemon_binary_path, default_engine_for_platform, detect_host_gfx_target, - detect_host_gpu_summary, engine_plugin_dirs, format_host_port, format_http_base_url, - interactive_terminal, load_model_recipe_registry, load_recent_automation_events, - managed_pip_cache_dir, model_artifact_cache_status, model_catalog_platforms, - model_recipe_featured, model_recipe_target_platform_label, normalize_therock_family, - platform_matches_gfx_family, process_is_running, read_http_response_bounded, uv_cache_source, - write_all_tcp_stream, + TELEMETRY_MODE_OFF, WatcherMode, append_audit_event, builtin_model_recipes, daemon_binary_path, + default_engine_for_platform, detect_host_gfx_target, detect_host_gpu_summary, + engine_plugin_dirs, interactive_terminal, load_model_recipe_registry, + load_recent_automation_events, managed_pip_cache_dir, model_artifact_cache_status, + model_catalog_platforms, model_recipe_featured, model_recipe_target_platform_label, + normalize_therock_family, platform_matches_gfx_family, process_is_running, uv_cache_source, }; use rocm_engine_protocol::{ DEFAULT_LOG_TAIL_LINES, DetectRequest, DetectResponse, DevicePolicy, - ENGINE_RECIPE_CONTRACT_VERSION, EngineMethod, EngineRecipeHint, GpuSelection, StopRequest, - StopResponse, + ENGINE_RECIPE_CONTRACT_VERSION, EngineMethod, GpuSelection, StopRequest, StopResponse, }; use std::collections::{BTreeMap, HashMap, VecDeque}; -use std::ffi::OsString; use std::fmt::Write as _; use std::fs; use std::io::{self, BufRead, Read, Write}; use std::path::{Path, PathBuf}; -use std::process::Command as ProcessCommand; use std::process::ExitCode; -#[cfg(all(test, not(windows)))] -use std::process::ExitStatus; #[cfg(not(windows))] use std::process::Stdio; +#[cfg(all(test, not(windows)))] +use std::process::{Command as ProcessCommand, ExitStatus}; #[cfg(test)] use std::sync::Mutex; use std::thread; @@ -7527,318 +7535,6 @@ impl From for WatcherMode { } } -const fn device_policy_name(policy: &DevicePolicy) -> &'static str { - match policy { - DevicePolicy::GpuRequired => "gpu_required", - DevicePolicy::GpuPreferred => "gpu_preferred", - DevicePolicy::CpuOnly => "cpu_only", - } -} - -#[derive(Debug, Clone)] -struct EngineSelection { - runtime_id: Option, - env_id: Option, - source: Option, -} - -fn resolve_engine_selection( - config: &RocmCliConfig, - engine: &str, - runtime_id: Option<&str>, - env_id: Option<&str>, -) -> EngineSelection { - if let Some(env_id) = env_id { - return EngineSelection { - runtime_id: None, - env_id: Some(env_id.to_owned()), - source: Some("cli_env_id".to_owned()), - }; - } - if let Some(runtime_id) = runtime_id { - return EngineSelection { - runtime_id: Some(runtime_id.to_owned()), - env_id: None, - source: Some("cli_runtime_id".to_owned()), - }; - } - - if let Some(runtime_key) = config.active_runtime_key.as_ref() { - return EngineSelection { - runtime_id: Some(runtime_key.clone()), - env_id: None, - source: Some("config_active_runtime_key".to_owned()), - }; - } - - if let Some(entry) = config.engine_config(engine) { - if let Some(env_id) = entry.preferred_env_id.as_ref() { - return EngineSelection { - runtime_id: None, - env_id: Some(env_id.clone()), - source: Some("config_preferred_env_id".to_owned()), - }; - } - if let Some(runtime_id) = entry.preferred_runtime_id.as_ref() { - return EngineSelection { - runtime_id: Some(runtime_id.clone()), - env_id: None, - source: Some("config_preferred_runtime_id".to_owned()), - }; - } - if let Some(env_id) = entry.last_installed_env_id.as_ref() { - return EngineSelection { - runtime_id: None, - env_id: Some(env_id.clone()), - source: Some("config_last_installed_env_id".to_owned()), - }; - } - if let Some(runtime_id) = entry.last_installed_runtime_id.as_ref() { - return EngineSelection { - runtime_id: Some(runtime_id.clone()), - env_id: None, - source: Some("config_last_installed_runtime_id".to_owned()), - }; - } - } - - if let Some(runtime_id) = config.default_runtime_id.as_ref() { - return EngineSelection { - runtime_id: Some(runtime_id.clone()), - env_id: None, - source: Some("config_default_runtime_id".to_owned()), - }; - } - - EngineSelection { - runtime_id: None, - env_id: None, - source: None, - } -} - -fn validate_engine_selection_runtime( - paths: &AppPaths, - mut selection: EngineSelection, -) -> Result { - if let Some(runtime_id) = selection.runtime_id.as_deref() { - let source = selection.source.as_deref().unwrap_or("runtime selection"); - selection.runtime_id = Some(resolve_runtime_selector_to_exact_key( - paths, runtime_id, source, - )?); - } else if selection.env_id.is_none() - && let Some(runtime_key) = single_ready_runtime_key(paths)? - { - selection.runtime_id = Some(runtime_key); - selection.source = Some("single_ready_runtime".to_owned()); - } - Ok(selection) -} - -fn single_ready_runtime_key(paths: &AppPaths) -> Result> { - let config = RocmCliConfig::load(paths).unwrap_or_default(); - recover_setup_runtime_registration(paths, &config)?; - let manifests = therock::load_runtime_manifests(paths)?; - let ready = manifests - .iter() - .filter(|manifest| validate_runtime_manifest_for_activation(manifest).is_ok()) - .collect::>(); - Ok(match ready.as_slice() { - [manifest] => Some(manifest.runtime_key.clone()), - _ => None, - }) -} - -fn resolve_runtime_selector_to_exact_key( - paths: &AppPaths, - selector: &str, - source: &str, -) -> Result { - let manifests = therock::load_runtime_manifests(paths)?; - match select_runtime_manifest(&manifests, selector) { - Ok(manifest) => Ok(manifest.runtime_key.clone()), - Err(error) => { - let config = RocmCliConfig::load(paths).unwrap_or_default(); - if recover_setup_runtime_registration(paths, &config)?.is_some() { - let manifests = therock::load_runtime_manifests(paths)?; - if let Ok(manifest) = select_runtime_manifest(&manifests, selector) { - return Ok(manifest.runtime_key.clone()); - } - } - bail!( - "runtime selector `{selector}` from {source} is not an exact usable runtime: {error}; run `rocm runtimes list` and `rocm runtimes activate `, or pass --runtime-id " - ) - } - } -} - -fn optional_arg(flag: &str, value: Option<&str>) -> Vec { - match value { - Some(value) => vec![flag.to_owned(), value.to_owned()], - None => Vec::new(), - } -} - -fn engine_recipe_json_arg(engine_recipe: Option<&EngineRecipeHint>) -> Result> { - match engine_recipe { - Some(engine_recipe) => Ok(vec![ - "--engine-recipe-json".to_owned(), - serde_json::to_string(engine_recipe).context("failed to encode engine recipe hint")?, - ]), - None => Ok(Vec::new()), - } -} - -fn parse_engine_recipe_json_arg(value: Option) -> Result> { - value - .map(|value| { - serde_json::from_str(&value).context("failed to parse --engine-recipe-json payload") - }) - .transpose() -} - -fn app_path_env_vars(paths: &AppPaths) -> [(&'static str, &Path); 3] { - [ - ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), - ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), - ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), - ] -} - -#[cfg_attr(not(windows), allow(dead_code))] -fn app_path_env_var_values( - paths: &AppPaths, - env_root: Option<&Path>, -) -> Vec<(&'static str, PathBuf)> { - let mut vars = app_path_env_vars(paths) - .into_iter() - .map(|(key, value)| (key, value.to_path_buf())) - .collect::>(); - if let Some(env_root) = env_root { - vars.push(("ROCM_CLI_ENGINE_ENVS_ROOT", env_root.to_path_buf())); - } - vars -} - -#[cfg_attr(not(windows), allow(dead_code))] -fn app_path_env_var_refs<'a>(vars: &'a [(&'static str, PathBuf)]) -> Vec<(&'static str, &'a Path)> { - vars.iter() - .map(|(key, value)| (*key, value.as_path())) - .collect() -} - -/// Argv passed to the embedded `rocmd` library to run the real foreground -/// automation loop (the same path as `rocmd run --automations-enabled`). -fn daemon_run_argv() -> Vec { - vec![ - OsString::from("rocmd"), - OsString::from("run"), - OsString::from("--automations-enabled"), - ] -} - -fn managed_service_launcher_path() -> Result { - let current_exe = daemon_binary_path()?; - if rocm_core::runtime_is_windows() { - return Ok(rocm_core::normalize_runtime_path_for_storage(¤t_exe)); - } - Ok(current_exe) -} - -fn apply_app_path_env(command: &mut ProcessCommand, paths: &AppPaths) { - for (key, value) in app_path_env_vars(paths) { - command.env(key, value); - } -} - -fn wait_for_service_http_ready( - engine: &str, - host: &str, - port: u16, - canonical_model_id: &str, - endpoint_api_key: Option<&str>, - timeout: Duration, -) -> EndpointReadiness { - wait_for_service_http_ready_with_progress( - engine, - host, - port, - &|| canonical_model_id.to_owned(), - endpoint_api_key, - timeout, - &mut |_elapsed| {}, - ) -} - -/// Wait until the service can actually serve, reporting how far it got. -/// -/// A health or model-listing endpoint answering is not enough to call a service -/// ready: engines advertise a model within seconds of accepting its name, while -/// the weights can take minutes to become usable, so a caller that sends its -/// first request on that signal gets a hang. Only [`EndpointReadiness::Serving`] -/// — an inference request that came back — means ready; a service that lists the -/// model without answering one is [`EndpointReadiness::Listing`], and the best -/// result seen before `timeout` is what gets returned. -/// -/// Polls until `timeout` elapses, invoking `on_tick(elapsed)` once per iteration -/// so a caller can animate a spinner. Engine-neutral: `service_http_readiness_paths` -/// maps each engine to the right listing path. `endpoint_api_key` is sent as a -/// bearer token so both the listing check and the inference probe still succeed -/// against a public endpoint that requires authentication. -fn wait_for_service_http_ready_with_progress( - engine: &str, - host: &str, - port: u16, - model_name: &dyn Fn() -> String, - endpoint_api_key: Option<&str>, - timeout: Duration, - on_tick: &mut dyn FnMut(Duration), -) -> EndpointReadiness { - let start = std::time::Instant::now(); - let endpoint = format_http_base_url(host, port); - let mut best = EndpointReadiness::Unreachable; - while start.elapsed() < timeout { - // Re-evaluated each pass: a container engine reports the name its server - // answers to only after it starts, so a name fixed up front can be wrong. - let canonical_model_id = model_name(); - let canonical_model_id = canonical_model_id.as_str(); - let listed = service_http_readiness_paths(engine).iter().any(|path| { - http_get_local_service( - host, - port, - path, - endpoint_api_key, - Duration::from_millis(750), - ) - .is_ok_and(|(status, body)| { - service_http_readiness_response_ready( - engine, - path, - status, - &body, - canonical_model_id, - ) - }) - }); - if listed { - best = EndpointReadiness::Listing; - if rocm_core::openai_chat_completion_probe( - &endpoint, - canonical_model_id, - endpoint_api_key, - rocm_core::INFERENCE_PROBE_TIMEOUT, - ) - .unwrap_or(false) - { - return EndpointReadiness::Serving; - } - } - on_tick(start.elapsed()); - thread::sleep(Duration::from_millis(250)); - } - best -} - /// The record status matching how far the endpoint got. /// /// `Listing` maps to `running`, not `starting`: the engine is up and loading @@ -7852,177 +7548,12 @@ const fn status_for_readiness(readiness: EndpointReadiness) -> &'static str { } } -fn service_http_readiness_paths(engine: &str) -> &'static [&'static str] { - match engine { - "lemonade" => &["/v1/health", "/v1/models"], - _ => &["/v1/models", "/v1/health", "/health", "/healthz"], - } -} - -fn http_get_local_service( - host: &str, - port: u16, - path: &str, - endpoint_api_key: Option<&str>, - timeout: Duration, -) -> Result<(u16, String)> { - let deadline = std::time::Instant::now() + timeout; - let mut stream = connect_tcp_stream(host, port, timeout)?; - let host_header = format_host_port(host, port); - // Authenticate the probe when the endpoint is protected; loopback endpoints - // pass `None` and the header is omitted. - let auth_header = match endpoint_api_key { - Some(key) => format!("Authorization: Bearer {key}\r\n"), - None => String::new(), - }; - let request = format!( - "GET {path} HTTP/1.1\r\nHost: {host_header}\r\n{auth_header}Connection: close\r\n\r\n" - ); - write_all_tcp_stream(&mut stream, request.as_bytes()) - .context("failed to write service readiness request")?; - let response = read_http_response_bounded(&mut stream, deadline) - .context("failed to read service readiness response")?; - let (headers, body) = response - .split_once("\r\n\r\n") - .unwrap_or((response.as_str(), "")); - let status = headers - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); - Ok((status, body.to_owned())) -} - -fn http_post_local_service_json( - host: &str, - port: u16, - path: &str, - body: &serde_json::Value, - timeout: Duration, -) -> Result<(u16, String)> { - let deadline = std::time::Instant::now() + timeout; - let mut stream = connect_tcp_stream(host, port, timeout)?; - let host_header = format_host_port(host, port); - let body = serde_json::to_string(body).context("failed to serialize service request")?; - let request = format!( - "POST {path} HTTP/1.1\r\nHost: {host_header}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - write_all_tcp_stream(&mut stream, request.as_bytes()) - .context("failed to write service request")?; - let response = read_http_response_bounded(&mut stream, deadline) - .context("failed to read service response")?; - let (headers, body) = response - .split_once("\r\n\r\n") - .unwrap_or((response.as_str(), "")); - let status = headers - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); - Ok((status, body.to_owned())) -} - -fn service_http_readiness_response_ready( - engine: &str, - path: &str, - status: u16, - body: &str, - canonical_model_id: &str, -) -> bool { - if status != 200 { - return false; - } - match (engine, path) { - ("lemonade", "/v1/health") => lemonade_health_ready_for_model(body, canonical_model_id), - ("lemonade", "/v1/models") => model_list_ready_for_model(body, canonical_model_id, true), - (_, "/v1/models") => model_list_ready_for_model(body, canonical_model_id, false), - _ => false, - } -} - -fn lemonade_health_ready_for_model(body: &str, canonical_model_id: &str) -> bool { - let Ok(value) = serde_json::from_str::(body.trim()) else { - return false; - }; - value - .get("all_models_loaded") - .and_then(serde_json::Value::as_array) - .is_some_and(|models| { - models.iter().any(|model| { - let name_matches = ["model_name", "id", "name"] - .into_iter() - .filter_map(|field| model.get(field).and_then(serde_json::Value::as_str)) - .any(|loaded| service_model_names_match(loaded, canonical_model_id)); - name_matches && service_model_reports_rocm_backend(model) - }) - }) -} - -fn model_list_ready_for_model( - body: &str, - canonical_model_id: &str, - require_rocm_backend: bool, -) -> bool { - let Ok(value) = serde_json::from_str::(body.trim()) else { - return false; - }; - value - .get("data") - .and_then(serde_json::Value::as_array) - .is_some_and(|models| { - models.iter().any(|model| { - let name_matches = ["id", "model", "name"] - .into_iter() - .filter_map(|field| model.get(field).and_then(serde_json::Value::as_str)) - .any(|loaded| service_model_names_match(loaded, canonical_model_id)); - name_matches && (!require_rocm_backend || service_model_gpu_ready(model)) - }) - }) -} - -/// Whether a lemonade `/v1/models` entry is served on GPU. A stock `llama-server` -/// (direct-serve) entry has no `recipe_options` and is accepted (that path only runs -/// GPU backends); a Lemonade-router entry carries `recipe_options`, so it must report a -/// ROCm backend — which keeps a registered-but-unloaded model (empty `recipe_options`) -/// from reading as ready. -fn service_model_gpu_ready(model: &serde_json::Value) -> bool { - match model.get("recipe_options") { - None => true, - Some(_) => service_model_reports_rocm_backend(model), - } -} - -fn service_model_reports_rocm_backend(model: &serde_json::Value) -> bool { - model - .get("recipe_options") - .and_then(|options| options.get("llamacpp_backend")) - .or_else(|| model.get("llamacpp_backend")) - .and_then(serde_json::Value::as_str) - .is_some_and(|backend| backend.trim().to_ascii_lowercase().starts_with("rocm")) -} - -fn service_model_names_match(left: &str, right: &str) -> bool { - let left = left.trim(); - let right = right.trim(); - if left.is_empty() || right.is_empty() { - return false; - } - if left.eq_ignore_ascii_case(right) { - return true; +const fn device_policy_name(policy: &DevicePolicy) -> &'static str { + match policy { + DevicePolicy::GpuRequired => "gpu_required", + DevicePolicy::GpuPreferred => "gpu_preferred", + DevicePolicy::CpuOnly => "cpu_only", } - let left = left - .trim_end_matches(".gguf") - .trim_end_matches(".safetensors") - .to_ascii_lowercase(); - let right = right - .trim_end_matches(".gguf") - .trim_end_matches(".safetensors") - .to_ascii_lowercase(); - left.contains(&right) || right.contains(&left) } fn treat_as_natural_language(args: &[String]) -> bool { @@ -8384,7 +7915,6 @@ pub(crate) mod tests { } use super::*; - use serde_json::json; use crate::runtime_install_finalize::tests::finish_sdk_install_with; use crate::runtimes_cmd::runtime_manifest_path; @@ -9459,144 +8989,6 @@ pub(crate) mod tests { ); } - #[test] - fn service_http_readiness_requires_loaded_lemonade_model() { - let loading = json!({ "all_models_loaded": [] }).to_string(); - assert!(!service_http_readiness_response_ready( - "lemonade", - "/v1/health", - 200, - &loading, - "Qwen3-0.6B-GGUF" - )); - - let loaded = json!({ - "all_models_loaded": [{ - "model_name": "Qwen3-0.6B-GGUF", - "recipe_options": { "llamacpp_backend": "rocm" } - }] - }) - .to_string(); - assert!(service_http_readiness_response_ready( - "lemonade", - "/v1/health", - 200, - &loaded, - "Qwen3-0.6B-GGUF" - )); - - let loaded_cpu = json!({ - "all_models_loaded": [{ - "model_name": "Qwen3-0.6B-GGUF", - "recipe_options": { "llamacpp_backend": "cpu" } - }] - }) - .to_string(); - assert!(!service_http_readiness_response_ready( - "lemonade", - "/v1/health", - 200, - &loaded_cpu, - "Qwen3-0.6B-GGUF" - )); - } - - #[test] - fn service_http_readiness_requires_model_list_entry() { - let empty = json!({ "data": [] }).to_string(); - assert!(!service_http_readiness_response_ready( - "vllm", - "/v1/models", - 200, - &empty, - "tiny.gguf" - )); - - let models = json!({ "data": [{ "id": "tiny.gguf" }] }).to_string(); - assert!(service_http_readiness_response_ready( - "vllm", - "/v1/models", - 200, - &models, - "tiny.gguf" - )); - - let lemonade_cpu_models = json!({ - "data": [{ - "id": "Qwen3-0.6B-GGUF", - "recipe_options": { "llamacpp_backend": "cpu" } - }] - }) - .to_string(); - assert!(!service_http_readiness_response_ready( - "lemonade", - "/v1/models", - 200, - &lemonade_cpu_models, - "Qwen3-0.6B-GGUF" - )); - - let lemonade_rocm_models = json!({ - "data": [{ - "id": "Qwen3-0.6B-GGUF", - "recipe_options": { "llamacpp_backend": "rocm" } - }] - }) - .to_string(); - assert!(service_http_readiness_response_ready( - "lemonade", - "/v1/models", - 200, - &lemonade_rocm_models, - "Qwen3-0.6B-GGUF" - )); - - assert!(!service_http_readiness_response_ready( - "vllm", - "/health", - 200, - "OK", - "tiny.gguf" - )); - assert!(!service_http_readiness_response_ready( - "vllm", - "/healthz", - 200, - "OK", - "Qwen3-0.6B-GGUF" - )); - } - - #[test] - fn lemonade_direct_serve_model_reads_ready_without_recipe_options() { - // The HF direct-serve path runs a stock llama-server whose `/v1/models` entry - // has no `recipe_options`. It must read as ready by name (that path is GPU-only), - // while a registered-but-unloaded lemonade entry (empty `recipe_options`) must not. - let direct = json!({ - "data": [{ "id": "LiquidAI/LFM2.5-230M-GGUF:Q4_0", "object": "model" }] - }) - .to_string(); - assert!(service_http_readiness_response_ready( - "lemonade", - "/v1/models", - 200, - &direct, - "LiquidAI/LFM2.5-230M-GGUF:Q4_0" - )); - - let registered = json!({ - "data": [{ "id": "LiquidAI/LFM2.5-230M-GGUF:Q4_0", "recipe_options": {} }] - }) - .to_string(); - assert!(!service_http_readiness_response_ready( - "lemonade", - "/v1/models", - 200, - ®istered, - "LiquidAI/LFM2.5-230M-GGUF:Q4_0" - )); - } - #[test] fn lemonade_stop_unloads_selected_model_over_http() -> Result<()> { use std::io::{Read, Write}; @@ -9724,175 +9116,6 @@ pub(crate) mod tests { Ok(()) } - #[test] - fn serve_readiness_wait_withholds_ready_while_the_model_only_lists() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; - - // This is the wait behind `rocm serve`, and its verdict is what - // `services list` later prints. A model listing on `/v1/models` while - // inference still fails must come back as `Listing` — reporting it ready - // is the false positive users and automation trip over. - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let port = listener.local_addr()?.port(); - let server = thread::spawn(move || -> Result<()> { - // Serve until the wait below gives up and the test drops the socket. - while let Ok((mut stream, _)) = listener.accept() { - stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); - let mut buffer = [0_u8; 1024]; - let Ok(read) = stream.read(&mut buffer) else { - continue; - }; - let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); - let (status_line, body) = if request.starts_with("POST /v1/chat/completions ") { - ("HTTP/1.1 503 Service Unavailable", r#"{"error":"loading"}"#) - } else { - ("HTTP/1.1 200 OK", r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"#) - }; - let _ = write!( - stream, - "{status_line}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - } - Ok(()) - }); - - let readiness = wait_for_service_http_ready( - "vllm", - "127.0.0.1", - port, - "Qwen3-0.6B-GGUF", - None, - Duration::from_millis(600), - ); - - assert_eq!( - readiness, - EndpointReadiness::Listing, - "a listed-but-unservable model is not ready" - ); - assert_eq!( - status_for_readiness(readiness), - "running", - "a loading service must not be recorded as `starting`, which rocmd \ - restarts once stale" - ); - drop(server); - Ok(()) - } - - #[test] - fn serve_readiness_wait_reports_ready_once_inference_answers() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; - - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let port = listener.local_addr()?.port(); - let server = thread::spawn(move || -> Result<()> { - while let Ok((mut stream, _)) = listener.accept() { - stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); - let mut buffer = [0_u8; 1024]; - let Ok(read) = stream.read(&mut buffer) else { - continue; - }; - let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); - let body = if request.starts_with("POST /v1/chat/completions ") { - r#"{"choices":[{"message":{"content":"ok"}}]}"# - } else { - r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"# - }; - let _ = write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - } - Ok(()) - }); - - let readiness = wait_for_service_http_ready( - "vllm", - "127.0.0.1", - port, - "Qwen3-0.6B-GGUF", - None, - Duration::from_secs(5), - ); - - assert_eq!(readiness, EndpointReadiness::Serving); - assert_eq!(status_for_readiness(readiness), "ready"); - drop(server); - Ok(()) - } - - #[test] - fn serve_readiness_ready_verdict_does_not_wait_for_the_peer_to_close() -> Result<()> { - use std::io::{Read, Write}; - use std::net::TcpListener; - use std::time::Instant; - - // Regression test for the other half of this PR's fix: a response is - // read to completion by its own framing, not by waiting for the peer - // to close. Before this fix, `read_tcp_stream_to_string` blocked - // until EOF, so a keep-alive engine that answers correctly but never - // closes the socket looked identical to a hung one — the readiness - // probe ran out its timeout and reported not-ready even though the - // answer had already arrived. - let listener = TcpListener::bind(("127.0.0.1", 0))?; - let port = listener.local_addr()?.port(); - let server = thread::spawn(move || { - while let Ok((mut stream, _)) = listener.accept() { - thread::spawn(move || { - stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); - let mut buffer = [0_u8; 1024]; - let Ok(read) = stream.read(&mut buffer) else { - return; - }; - let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); - let body = if request.starts_with("POST /v1/chat/completions ") { - r#"{"choices":[{"message":{"content":"ok"}}]}"# - } else { - r#"{"data":[{"id":"Qwen3-0.6B-GGUF"}]}"# - }; - let _ = write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", - body.len(), - body - ); - // Hold the connection open well past the readiness wait's - // timeout below, and deliberately omit `Connection: - // close`. The client must not need EOF to recognize the - // response as complete. - thread::sleep(Duration::from_secs(10)); - }); - } - }); - - let started = Instant::now(); - let readiness = wait_for_service_http_ready( - "vllm", - "127.0.0.1", - port, - "Qwen3-0.6B-GGUF", - None, - Duration::from_secs(5), - ); - - assert_eq!(readiness, EndpointReadiness::Serving); - assert!( - started.elapsed() < Duration::from_secs(3), - "a complete response must be recognized without waiting on the peer to close" - ); - // `server`'s accept loop runs forever; dropping the JoinHandle detaches - // it rather than joining, and the thread dies with the test process. - drop(server); - Ok(()) - } - pub(crate) fn test_app_paths() -> AppPaths { AppPaths { config_dir: PathBuf::from("C:/Users/test/.rocm"), @@ -9901,28 +9124,6 @@ pub(crate) mod tests { } } - #[test] - fn app_path_env_vars_include_config_data_and_cache() { - let paths = test_app_paths(); - let vars = app_path_env_vars(&paths); - - assert_eq!(vars[0], ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path())); - assert_eq!(vars[1], ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path())); - assert_eq!(vars[2], ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path())); - } - - #[test] - fn app_path_env_var_values_include_engine_env_root_when_needed() { - let paths = test_app_paths(); - let engine_root = PathBuf::from("D:/rocm-data/runtime/engines"); - let vars = app_path_env_var_values(&paths, Some(&engine_root)); - - assert_eq!( - vars.last().map(|(key, value)| (*key, value.as_path())), - Some(("ROCM_CLI_ENGINE_ENVS_ROOT", engine_root.as_path())) - ); - } - #[test] fn freeform_invocation_supports_leading_yes_for_natural_language_only() { let invocation = parse_freeform_invocation(&[ @@ -12752,111 +11953,6 @@ pub(crate) mod tests { ); } - #[test] - fn resolve_engine_selection_uses_default_runtime_after_engine_prefs() { - let mut config = RocmCliConfig { - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..RocmCliConfig::default() - }; - - let selection = resolve_engine_selection(&config, "vllm", None, None); - assert_eq!( - selection.runtime_id.as_deref(), - Some("therock-release:gfx120X-all") - ); - assert_eq!( - selection.source.as_deref(), - Some("config_default_runtime_id") - ); - - config.active_runtime_key = Some("release-pip-gfx120x-all-7-13-0".to_owned()); - let selection = resolve_engine_selection(&config, "vllm", None, None); - assert_eq!( - selection.runtime_id.as_deref(), - Some("release-pip-gfx120x-all-7-13-0") - ); - assert_eq!( - selection.source.as_deref(), - Some("config_active_runtime_key") - ); - - config.engine_config_mut("vllm").preferred_runtime_id = - Some("therock-nightly:gfx120X-all".to_owned()); - let selection = resolve_engine_selection(&config, "vllm", None, None); - assert_eq!( - selection.runtime_id.as_deref(), - Some("release-pip-gfx120x-all-7-13-0") - ); - assert_eq!( - selection.source.as_deref(), - Some("config_active_runtime_key") - ); - - config.active_runtime_key = None; - let selection = resolve_engine_selection(&config, "vllm", None, None); - assert_eq!( - selection.runtime_id.as_deref(), - Some("therock-nightly:gfx120X-all") - ); - assert_eq!( - selection.source.as_deref(), - Some("config_preferred_runtime_id") - ); - } - - #[test] - fn engine_selection_uses_single_ready_runtime_without_active_marker() -> Result<()> { - let (root, paths) = test_paths("single-ready-runtime-selection"); - let manifest = write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-14-0", - "therock-release:gfx120X-all", - "7.14.0", - 20, - )?; - let selection = validate_engine_selection_runtime( - &paths, - resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), - )?; - - assert_eq!( - selection.runtime_id.as_deref(), - Some(manifest.runtime_key.as_str()) - ); - assert_eq!(selection.source.as_deref(), Some("single_ready_runtime")); - let _ = fs::remove_dir_all(root); - Ok(()) - } - - #[test] - fn engine_selection_keeps_multiple_ready_runtimes_explicit() -> Result<()> { - let (root, paths) = test_paths("multiple-ready-runtime-selection"); - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-13-0", - "therock-release:gfx120X-all", - "7.13.0", - 10, - )?; - write_test_pip_runtime( - &paths, - "release-pip-gfx120x-all-7-14-0", - "therock-release:gfx120X-all", - "7.14.0", - 20, - )?; - let selection = validate_engine_selection_runtime( - &paths, - resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), - )?; - - assert!(selection.runtime_id.is_none()); - assert!(selection.env_id.is_none()); - assert!(selection.source.is_none()); - let _ = fs::remove_dir_all(root); - Ok(()) - } - #[test] fn render_config_text_includes_default_runtime() { let (_root, paths) = test_paths("config-default-runtime"); diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 27545eb138..1d948ef976 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -1820,6 +1820,8 @@ fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> #[cfg(test)] pub(crate) mod tests { use super::*; + use crate::append_examine_runtime_state; + use crate::engine_runtime_select::resolve_runtime_selector_to_exact_key; use crate::runtime_install_finalize::{ finalize_successful_sdk_install, render_sdk_install_success, }; @@ -1827,7 +1829,6 @@ pub(crate) mod tests { use crate::tests::{ NEW_RUNTIME_KEY, OLD_RUNTIME_KEY, plant_service_record_on_runtime, test_paths, }; - use crate::{append_examine_runtime_state, resolve_runtime_selector_to_exact_key}; #[test] fn runtime_selector_recovers_setup_runtime_registry_from_local_manifest() -> Result<()> { diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 2d76556281..22f02bd0ef 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -16,8 +16,10 @@ //! `parse_device_policy`, `parse_gpu_selection`, //! `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, //! and `validate_pinned_gpu_index` moved out of `main.rs` into -//! `gpu_policy.rs` in ROCMAI-214 (Phase 6f); this file's `use` list below -//! points there now instead of at the crate root. The +//! `gpu_policy.rs` in ROCMAI-214 (Phase 6f); `resolve_engine_selection` and +//! `validate_engine_selection_runtime` moved out of `main.rs` into +//! `engine_runtime_select.rs` in ROCMAI-215 (Phase 6f); this file's `use` +//! list below points to both now instead of at the crate root. The //! managed-service-spawning tail this file calls into — `start_managed_service`, //! `run_attached_service`, `resolve_endpoint_auth`, //! `ensure_public_bind_engine_supported`, @@ -43,6 +45,7 @@ use rocm_engine_protocol::{ use crate::DevicePolicyArg; use crate::endpoint_keys; use crate::engine_ipc::engine_request; +use crate::engine_runtime_select::{resolve_engine_selection, validate_engine_selection_runtime}; use crate::engines_cmd::{engine_manages_own_runtime, ensure_self_managed_engine_ready}; use crate::gpu_policy::{ detect_gpu_count, gpu_vram_usage, parse_device_policy, parse_gpu_selection, @@ -54,10 +57,7 @@ use crate::service_launch::{ ensure_public_bind_engine_supported, print_managed_launch_plain, resolve_endpoint_auth, run_attached_service, start_managed_service, validate_bind_host, }; -use crate::{ - cli_progress, collect_serve_notes, device_policy_name, resolve_engine_selection, - validate_engine_selection_runtime, -}; +use crate::{cli_progress, collect_serve_notes, device_policy_name}; #[derive(Debug, Clone, Eq, PartialEq)] pub(crate) struct ServeEngineSelection { diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 46e6431c67..71074abde1 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -47,18 +47,22 @@ use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse} use crate::endpoint_keys; use crate::engine_ipc::builtin_engine_serve_http_args; +#[cfg(not(windows))] +use crate::engine_runtime_select::apply_app_path_env; +#[cfg(windows)] +use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; +use crate::engine_runtime_select::{ + managed_service_launcher_path, wait_for_service_http_ready_with_progress, +}; use crate::engines_cmd::env_root_for_service; use crate::managed_service_status::existing_live_managed_service; +#[cfg(not(windows))] +use crate::read_optional_tail_lines; use crate::service_hints; use crate::{ - SandboxToolArg, device_policy_name, managed_service_launcher_path, record_cli_audit_event, - render_service_action_result, run_internal_sandbox_tool, status_for_readiness, - wait_for_service_http_ready_with_progress, + SandboxToolArg, device_policy_name, record_cli_audit_event, render_service_action_result, + run_internal_sandbox_tool, status_for_readiness, }; -#[cfg(windows)] -use crate::{app_path_env_var_refs, app_path_env_var_values}; -#[cfg(not(windows))] -use crate::{apply_app_path_env, read_optional_tail_lines}; pub(crate) fn validate_bind_host(host: &str, allow_public_bind: bool) -> Result<()> { if !is_loopback_host(host) && !allow_public_bind { diff --git a/docs/architecture.md b/docs/architecture.md index 2a0f953f5b..35aa4a1862 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -41,6 +41,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`driver_install.rs`](../apps/rocm/src/driver_install.rs) — mechanical relocation, owns private types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6a split, ROCMAI-82; see the Phase 6a variant note above): `install_driver()`/`reconcile_driver_install()`, plus the driver-plan/state types. - [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) — mechanical relocation, dispatch-adjacent, no owned types. - [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-213): resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the stdio request/response envelope used to talk to an external engine binary or the in-process built-in handler (`engine_request`, `engine_request_with_env_root`, `decode_engine_response`, the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). Two items that sit physically inside this span in `main.rs` stayed there instead of moving: `missing_packaged_engine_reason` has callers outside the cluster (plus its own test), and `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request`. `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from `main.rs`; `engine_request` is also read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `engine_request_with_env_root` is `pub(crate)` and read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) (not `main.rs`); `builtin_engine_serve_http_args` is also read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs). `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `decode_engine_response`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. +- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-215, the last sub-phase of this split before `main.rs`'s single shared `mod tests` block begins): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`optional_arg`, `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it already had callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and `main.rs`'s own `restart_internal_managed_service_with`), so it stays a shared crate-root function, reached from here (and from this module's own tests) the same way other mechanically relocated modules reach a left-behind crate-root item, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. - [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) — mechanical relocation, owns private types (Phase 6a): `engines()`, plus env-root/runtime resolution — several helpers here are also called from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and from other root-level commands, so they stay `pub(crate)`. - [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) — mechanical relocation, dispatch-adjacent, owns a `pub(crate)` type (as of Phase 6e's follow-up, ROCMAI-726; see the Phase 6a variant note above for the owned-types pattern). Holds the freeform natural-language planner's plan-*execution*/consent half that [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) below deliberately left out: `validate_freeform_execution_action()`, `prepare_freeform_execution()`, `apply_freeform_execution_consent()`, `render_freeform_execution_header()`, and the `pub(crate)` `FreeformExecution` struct that `prepare_freeform_execution` returns and `render_freeform_execution_header` consumes. `main.rs`'s `execute_freeform_next_action()` and `run_freeform()` stay in `main.rs` — both are dispatch-level, not part of deciding whether a plan's next action is safe to execute — and `execute_freeform_next_action()` calls in via the unqualified re-import `use crate::freeform_execution::{prepare_freeform_execution, render_freeform_execution_header};` (`run_freeform()` only calls `execute_freeform_next_action()`, never this module directly), matching [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s dispatch-adjacent form rather than `freeform_planner.rs`'s own qualified-path form. `prepare_freeform_execution` takes the already-built `plan` by value rather than rebuilding it from `request`/`paths`/`config` (ROCMAI-725's single-build guarantee, inherited unchanged from `main.rs`); `prepare_freeform_execution`/`render_freeform_execution_header` are `pub(crate)` for that cross-module call; `FreeformExecution` was already `pub(crate)` (read directly by `main.rs` and this module's own tests); `validate_freeform_execution_action`/`apply_freeform_execution_consent` have no caller outside this module and stay module-private, unchanged from before the move. The cluster's six regression tests moved out of `main.rs`'s shared `mod tests` into this module's own `mod tests`, inheriting the qualified `freeform_planner::` path they already used to reach `freeform_planner.rs`'s items as `main.rs`'s former tests — this module reaches in from outside using the same qualified path `main.rs` and `providers.rs`'s tests use (unlike `chat.rs`/`comfyui.rs`, which import it unqualified). - [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) — full domain extraction (Phase 6e, ROCMAI-87). Owns the freeform natural-language planner cluster's types (`StructuredRequestPlan`, `PlannerIntent`, `PlannedToolCall`, `FreeformPlanAction`, `ProviderPlannerResponse`, `ProviderPlannerToolCall`, all module-private except `FreeformPlanAction` and the fields of `StructuredRequestPlan`/`PlannedToolCall` that `main.rs` reads directly), the deterministic plan builder (`build_freeform_plan*`), the provider-assisted path (`resolve_freeform_plan_with_provider`, `build_provider_planner_prompt`, `provider_planner_response_to_plan`, `validate_provider_planner_tool_call`), and plan rendering (`render_structured_request_plan`). [`main.rs`](../apps/rocm/src/main.rs) calls `build_freeform_plan_with_context`/`render_structured_request_plan` directly for the dispatch/review flow (`main.rs`'s only other call into this module, `render_launch_summary` → `host_default_engine`, is unrelated: bare `rocm`'s non-interactive launch summary), while [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) above calls `plan_next_action`/`format_structured_tool_call` for the execution flow and reads `FreeformPlanAction`/`StructuredRequestPlan` by type for its execution-validation functions — those live there, not here, because they decide whether a plan's next action is safe to *execute*, a dispatch-adjacent concern distinct from *building* the plan; `main.rs`'s `run_freeform` builds a plan exactly once via `build_freeform_plan_with_context` and threads that same value through `execute_freeform_next_action` to `prepare_freeform_execution` rather than rebuilding it (ROCMAI-725: building a plan can call a configured cloud `planner_provider`, which is not guaranteed deterministic, so a second independent build could silently disagree with the first). [`chat.rs`](../apps/rocm/src/chat.rs)'s `natural_language_plan` MCP tool gets the same single-build guarantee from `plan_freeform_request_with_context`, which builds once and returns both the rendered text and the next action; `chat.rs` also calls `format_structured_tool_call` directly, like [`comfyui.rs`](../apps/rocm/src/comfyui.rs) does, and its `validate_chat_rocm_command_safety` reads `serve_args_request_cpu_device` directly as a no-CPU-fallback safety check on provider-proposed `serve` args. `host_default_engine`, `build_freeform_plan_with_context`, `render_structured_request_plan`, `plan_next_action`, `plan_freeform_request_with_context`, `provider_planner_response_to_plan`, `format_structured_tool_call`, and `serve_args_request_cpu_device` are `pub(crate)` for these reasons; `freeform_plan_next_action` and `render_freeform_plan` are test-only (`#[cfg(test)]`) and `FreeformPlanAction` was already `pub(crate)` before the split. `parse_freeform_invocation`/`should_treat_as_freeform` (deciding whether an invocation is freeform at all) stay in `main.rs`, a dispatch-level concern outside both this module and `freeform_execution.rs`. The cluster's regression tests live in `freeform_planner.rs`'s own `mod tests`; the handful of `main.rs` tests that exercise the invocation classifier/clap-suggestion-avoidance stay in `main.rs`'s shared `mod tests`, since neither is specific to this module or to `freeform_execution.rs` (and neither touches `freeform_planner::` at all), and the ROCMAI-725 regression tests proving the single-build guarantee live in `providers.rs`'s `mod tests` (reusing its mock local-provider server), reaching this module's items through the qualified `freeform_planner::` path like any other external caller.