From 68bfa83aa463fd114daf70f1a1b8d295a0eb5559 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 15:19:23 +0000 Subject: [PATCH 1/4] refactor(providers): move the ROCm tool schema into providers/tool_schema.rs providers.rs mixes provider dispatch with the ROCm tool catalog it advertises to models. Move rocm_openai_tool_definitions, rocm_openai_tool and rocm_anthropic_tool_definitions into a leaf child module, providers/tool_schema.rs. This is a pure move: the bodies are byte-identical, and the two definition functions become pub(super) because the request-body builders that read them stay in providers.rs for now. rocm_openai_tool stays private. No tool is added, renamed or re-described, and no test changes (the tests that assert on the catalog go through the request-body builders and move with them later). docs/architecture.md gains the providers/tool_schema.rs entry and a fuller providers.rs entry. Signed-off-by: Jussi Elo --- apps/rocm/src/providers.rs | 185 +----------------------- apps/rocm/src/providers/tool_schema.rs | 191 +++++++++++++++++++++++++ docs/architecture.md | 3 +- 3 files changed, 196 insertions(+), 183 deletions(-) create mode 100644 apps/rocm/src/providers/tool_schema.rs diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index 1a77882e34..d271d2d114 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -2,6 +2,9 @@ // // SPDX-License-Identifier: MIT +mod tool_schema; + +use self::tool_schema::{rocm_anthropic_tool_definitions, rocm_openai_tool_definitions}; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AuditEventRecord, EndpointReadiness, EndpointReadinessOutcome, ManagedServiceRecord, @@ -1069,188 +1072,6 @@ fn openai_chat_request_body_with_stream( body } -fn rocm_openai_tool_definitions() -> Vec { - vec![ - rocm_openai_tool( - "examine", - "Read the current ROCm host, GPU, runtime, driver, and engine status.", - serde_json::json!({ - "type": "object", - "properties": {}, - "additionalProperties": false - }), - ), - rocm_openai_tool( - "bridge_snapshot", - "Read a full ROCm snapshot including examine data, engines, services, automations, and GPU telemetry.", - serde_json::json!({ - "type": "object", - "properties": {}, - "additionalProperties": false - }), - ), - rocm_openai_tool( - "gpu_snapshot", - "Read the current local AMD GPU telemetry snapshot when available.", - serde_json::json!({ - "type": "object", - "properties": {}, - "additionalProperties": false - }), - ), - rocm_openai_tool( - "engines", - "List available ROCm serving engines and whether each is installed.", - serde_json::json!({ - "type": "object", - "properties": {}, - "additionalProperties": false - }), - ), - rocm_openai_tool( - "services", - "List managed local model servers and their current status.", - serde_json::json!({ - "type": "object", - "properties": {}, - "additionalProperties": false - }), - ), - rocm_openai_tool( - "service_logs", - "Read the recent log output for a managed local model server.", - serde_json::json!({ - "type": "object", - "properties": { - "service_id": { "type": "string" }, - "lines": { "type": "integer", "minimum": 1, "maximum": 500 } - }, - "required": ["service_id"], - "additionalProperties": false - }), - ), - rocm_openai_tool( - "automations", - "List watcher and automation status plus recent automation events.", - serde_json::json!({ - "type": "object", - "properties": { - "event_limit": { "type": "integer", "minimum": 1, "maximum": 64 } - }, - "additionalProperties": false - }), - ), - rocm_openai_tool( - "natural_language_plan", - "Ask ROCm CLI to translate a natural-language ROCm request into a visible plan without executing it.", - serde_json::json!({ - "type": "object", - "properties": { - "request": { "type": "string" } - }, - "required": ["request"], - "additionalProperties": false - }), - ), - rocm_openai_tool( - "path_exists", - "Read whether an exact local filesystem path exists and whether its parent folder exists. Use this only to check user-provided install folders before requesting a ROCm install.", - serde_json::json!({ - "type": "object", - "properties": { - "path": { "type": "string" } - }, - "required": ["path"], - "additionalProperties": false - }), - ), - rocm_openai_tool( - "port_status", - "Read whether a local loopback TCP port is listening and whether a ROCm-managed service owns it. Use this for questions like whether something is running on port 8188 or the local model server port. Do not use it for non-local hosts.", - serde_json::json!({ - "type": "object", - "properties": { - "host": { "type": "string" }, - "port": { "type": "integer", "minimum": 1, "maximum": 65535 } - }, - "required": ["port"], - "additionalProperties": false - }), - ), - rocm_openai_tool( - "rocm_command", - "Run or request a supported ROCm CLI command with argv-style arguments. Known read-only inspection commands may run immediately. Commands that install, start, serve, stop, delete, or change ROCm state are paused for user review before they run. Use this tool for mutating TheRock setup, including --prefix PATH when the user gives an install folder, --build-date YYYY-MM-DD or --version VERSION, config changes, ComfyUI install/start, engine install, vLLM management, and local LLM serve actions; do not request shell commands, CPU execution, or public network binds.", - serde_json::json!({ - "type": "object", - "properties": { - "args": { - "type": "array", - "items": { "type": "string" }, - "minItems": 1, - "maxItems": 64 - }, - "reason": { "type": "string" } - }, - "required": ["args"], - "additionalProperties": false - }), - ), - rocm_openai_tool( - "update_check", - "Check whether ROCm CLI knows about newer TheRock runtime versions.", - serde_json::json!({ - "type": "object", - "properties": {}, - "additionalProperties": false - }), - ), - rocm_openai_tool( - "install_sdk_dry_run", - "Preview a TheRock SDK install without changing the machine.", - serde_json::json!({ - "type": "object", - "properties": { - "channel": { "type": "string", "enum": ["release", "nightly"] }, - "format": { "type": "string", "enum": ["wheel", "tarball"] }, - "prefix": { "type": "string" }, - "version": { "type": "string" }, - "build_date": { "type": "string" } - }, - "additionalProperties": false - }), - ), - ] -} - -fn rocm_openai_tool( - name: &str, - description: &str, - parameters: serde_json::Value, -) -> serde_json::Value { - serde_json::json!({ - "type": "function", - "function": { - "name": name, - "description": description, - "parameters": parameters, - } - }) -} - -fn rocm_anthropic_tool_definitions() -> Vec { - rocm_openai_tool_definitions() - .into_iter() - .filter_map(|tool| { - let function = tool.get("function")?; - Some(serde_json::json!({ - "name": function.get("name")?.clone(), - "description": function.get("description")?.clone(), - "input_schema": function.get("parameters")?.clone(), - })) - }) - .collect() -} - fn anthropic_chat_request_body(model: &str, request: &ChatRequest) -> serde_json::Value { anthropic_chat_request_body_with_stream(model, request, false) } diff --git a/apps/rocm/src/providers/tool_schema.rs b/apps/rocm/src/providers/tool_schema.rs new file mode 100644 index 0000000000..ee4fdff776 --- /dev/null +++ b/apps/rocm/src/providers/tool_schema.rs @@ -0,0 +1,191 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The ROCm tool schema the providers advertise to a model: the OpenAI +//! function-calling definitions ([`rocm_openai_tool_definitions`]) and the same +//! catalog reshaped for Anthropic's `tools` field ([`rocm_anthropic_tool_definitions`]). +//! +//! A leaf of the `providers` module: nothing here depends on the rest of it. + +pub(super) fn rocm_openai_tool_definitions() -> Vec { + vec![ + rocm_openai_tool( + "examine", + "Read the current ROCm host, GPU, runtime, driver, and engine status.", + serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }), + ), + rocm_openai_tool( + "bridge_snapshot", + "Read a full ROCm snapshot including examine data, engines, services, automations, and GPU telemetry.", + serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }), + ), + rocm_openai_tool( + "gpu_snapshot", + "Read the current local AMD GPU telemetry snapshot when available.", + serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }), + ), + rocm_openai_tool( + "engines", + "List available ROCm serving engines and whether each is installed.", + serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }), + ), + rocm_openai_tool( + "services", + "List managed local model servers and their current status.", + serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }), + ), + rocm_openai_tool( + "service_logs", + "Read the recent log output for a managed local model server.", + serde_json::json!({ + "type": "object", + "properties": { + "service_id": { "type": "string" }, + "lines": { "type": "integer", "minimum": 1, "maximum": 500 } + }, + "required": ["service_id"], + "additionalProperties": false + }), + ), + rocm_openai_tool( + "automations", + "List watcher and automation status plus recent automation events.", + serde_json::json!({ + "type": "object", + "properties": { + "event_limit": { "type": "integer", "minimum": 1, "maximum": 64 } + }, + "additionalProperties": false + }), + ), + rocm_openai_tool( + "natural_language_plan", + "Ask ROCm CLI to translate a natural-language ROCm request into a visible plan without executing it.", + serde_json::json!({ + "type": "object", + "properties": { + "request": { "type": "string" } + }, + "required": ["request"], + "additionalProperties": false + }), + ), + rocm_openai_tool( + "path_exists", + "Read whether an exact local filesystem path exists and whether its parent folder exists. Use this only to check user-provided install folders before requesting a ROCm install.", + serde_json::json!({ + "type": "object", + "properties": { + "path": { "type": "string" } + }, + "required": ["path"], + "additionalProperties": false + }), + ), + rocm_openai_tool( + "port_status", + "Read whether a local loopback TCP port is listening and whether a ROCm-managed service owns it. Use this for questions like whether something is running on port 8188 or the local model server port. Do not use it for non-local hosts.", + serde_json::json!({ + "type": "object", + "properties": { + "host": { "type": "string" }, + "port": { "type": "integer", "minimum": 1, "maximum": 65535 } + }, + "required": ["port"], + "additionalProperties": false + }), + ), + rocm_openai_tool( + "rocm_command", + "Run or request a supported ROCm CLI command with argv-style arguments. Known read-only inspection commands may run immediately. Commands that install, start, serve, stop, delete, or change ROCm state are paused for user review before they run. Use this tool for mutating TheRock setup, including --prefix PATH when the user gives an install folder, --build-date YYYY-MM-DD or --version VERSION, config changes, ComfyUI install/start, engine install, vLLM management, and local LLM serve actions; do not request shell commands, CPU execution, or public network binds.", + serde_json::json!({ + "type": "object", + "properties": { + "args": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "maxItems": 64 + }, + "reason": { "type": "string" } + }, + "required": ["args"], + "additionalProperties": false + }), + ), + rocm_openai_tool( + "update_check", + "Check whether ROCm CLI knows about newer TheRock runtime versions.", + serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }), + ), + rocm_openai_tool( + "install_sdk_dry_run", + "Preview a TheRock SDK install without changing the machine.", + serde_json::json!({ + "type": "object", + "properties": { + "channel": { "type": "string", "enum": ["release", "nightly"] }, + "format": { "type": "string", "enum": ["wheel", "tarball"] }, + "prefix": { "type": "string" }, + "version": { "type": "string" }, + "build_date": { "type": "string" } + }, + "additionalProperties": false + }), + ), + ] +} + +fn rocm_openai_tool( + name: &str, + description: &str, + parameters: serde_json::Value, +) -> serde_json::Value { + serde_json::json!({ + "type": "function", + "function": { + "name": name, + "description": description, + "parameters": parameters, + } + }) +} + +pub(super) fn rocm_anthropic_tool_definitions() -> Vec { + rocm_openai_tool_definitions() + .into_iter() + .filter_map(|tool| { + let function = tool.get("function")?; + Some(serde_json::json!({ + "name": function.get("name")?.clone(), + "description": function.get("description")?.clone(), + "input_schema": function.get("parameters")?.clone(), + })) + }) + .collect() +} diff --git a/docs/architecture.md b/docs/architecture.md index 4dd3148e6d..68b2d43ab4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -61,7 +61,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat.rs`](../apps/rocm/src/chat.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat.rs`](../apps/rocm/src/chat.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. - [`model_fit.rs`](../apps/rocm/src/model_fit.rs) — mechanical relocation, owns a `pub(crate)` type: answering whether a host can run a model for `rocm diagnose --model` (`assess_model_on_this_host`, `assess_model_with_host_paths`, `assess_model_for_host`), the accelerator-memory fact it rests on (`host_accelerator_memory`, `classify_accelerator_memory`, `examination_reports_amd_gpu`) and the platform gate that rules an engine out (`PlatformRuleOut`, `engine_ruled_out_by_platform`, `unsupported_here_for`). It is a module of its own, rather than part of the `diagnose` handler, so [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs), which reads `PlatformRuleOut` and `engine_ruled_out_by_platform`, does not depend on a command module. It depends on [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`select_serve_engine`) and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs), neither of which depends on it. `assess_model_on_this_host` (called from [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` are `pub(crate)`, and [`main.rs`](../apps/rocm/src/main.rs) re-imports the last two so `examine_report_text.rs` reaches them as `crate::`; every other item is private. The cluster's regression tests live in this module's own `mod tests`; the one that re-runs the test binary on a helper test by its full path (`engine_ruled_out_by_platform_gates_aim_on_docker_reachability`) names the helper's new path, `model_fit::tests::docker_unreachable_subprocess_helper`. - [`provider_keys.rs`](../apps/rocm/src/provider_keys.rs) — full domain extraction. -- [`providers.rs`](../apps/rocm/src/providers.rs) — full domain extraction (no dedicated command enum; invoked from the existing chat/config command flows). +- [`providers.rs`](../apps/rocm/src/providers.rs) — full domain extraction (no dedicated command enum; invoked from the existing chat/config command flows). Keeps the request/response types, the `ProviderAdapter` trait and the `provider_*` dispatch, the `LocalProvider` and `RemoteProvider` implementations, local-service selection (`select_local_chat_service`, `ready_local_services`), the remote model and endpoint resolution, the `ureq` remote transport and the `SseBytesEmitter` trait. The ROCm tool schema lives in a child module under `providers/`, below. +- [`providers/tool_schema.rs`](../apps/rocm/src/providers/tool_schema.rs) — full domain extraction, a leaf child of `providers.rs`: the ROCm tool catalog a provider advertises to a model, as OpenAI function definitions (`rocm_openai_tool_definitions`) and reshaped for Anthropic's `tools` field (`rocm_anthropic_tool_definitions`). Both are `pub(super)`, read by the request-body builders in `providers.rs`; `rocm_openai_tool`, the one-tool constructor behind them, is private. Depends on nothing else in `providers`. - [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. - [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install: the orchestration (`finish_sdk_install`, `settle_engine_install`, `finalize_successful_sdk_install`), the engine-dependency check, the runtime device check and the post-install verdict built on it, and `SdkInstallConsents`. The torch alignment and retention it runs live in the child module [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs). `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`; `settle_engine_install` is `pub(crate)` and read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs). The host system-package installs live in [`system_packages.rs`](../apps/rocm/src/system_packages.rs); this module re-exports `ensure_libatomic_for_torch`/`ensure_libnuma_for_torch` (called from both `install_cmd.rs`'s `install()` and `engines_cmd.rs`) and `ensure_openmpi_for_vllm` (called from `engines_cmd.rs` and, for the engine auto-install after an SDK install, from here) so those callers keep their imports. - [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) — full domain extraction, a child of `runtime_install_finalize.rs` that depends on its parent's device check (`RuntimeDeviceCheck`, `device_check_verdict`) and on `runtime_manifest_for_selector` in [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): what to do about the torch an engine install finds in a managed runtime. `plan_torch_alignment` decides whether the SDK's torch must be realigned to the engine's pin, `align_runtime_torch` runs the realignment, `classify_retained_torch` decides when a torch that already works is kept as an intended divergence instead, and the `render_*` and `report_*` functions with `TorchAlignment` and `TorchRetention` print and log the outcome. The manifest readers (`sdk_torch_build_for_key`, `sdk_torch_build_from_manifest`, `runtime_python_for_key`, `runtime_library_paths_for_key`, `runtime_index_url_for_key`) live here too. The parent stays the caller (`settle_engine_install` and `maybe_auto_install_sdk_preferred_engine`), so it could not be a crate-level sibling without a cycle with the parent's device check. The items the parent calls are `pub(super)`: `TorchAlignment`, `TorchRetention`, `report_torch_alignment`, `report_torch_retention`, `classify_retained_torch`, `retained_diverged_package`, `deliberately_diverged_package`, `sdk_torch_build_for_key`, `runtime_python_for_key` and `runtime_library_paths_for_key`; the rest is private. Its tests cover planning, rendering, the opt-out and the retention rules, and reach the parent's private engine-dependency check as a descendant of it. From 5a921a3a7fd69fa966e7216200b0fd48ac61e436 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 15:24:16 +0000 Subject: [PATCH 2/4] refactor(providers): move the local HTTP/SSE transport into providers/local_http.rs The local provider talks to an engine over a hand-rolled HTTP/1.1 client: a blocking JSON POST and a streamed POST that decodes chunked, Content-Length and close-delimited SSE bodies. It sat in providers.rs next to provider dispatch and the wire formats. Move it into a child module, providers/local_http.rs: post_json_to_local_endpoint, local_bearer_header, post_json_to_local_endpoint_body, stream_json_from_local_endpoint, read_sse_body, header_value, read_fixed_sse_body, read_close_delimited_sse_body, LOCAL_STREAM_TRUNCATED, body_truncated_at_eof, read_chunked_sse_body and parse_http_endpoint. This is a pure move: the bodies are byte-identical. The two entry points and LOCAL_STREAM_TRUNCATED (which providers.rs's tests read) become pub(super); the rest stays private. providers/proptests.rs moves to providers/local_http/proptests.rs and is declared from local_http.rs, so the property tests of the SSE readers sit with the readers; the one example test of parse_http_endpoint moves with them. The tests that drive LocalProvider against a mock server stay in providers.rs with their server helpers. No test is added, removed or renamed. local_http reaches OpenAiSseEmitter, SseBytesEmitter and ProviderStreamEvent through super; nothing in providers depends back on its private items. docs/architecture.md gains the providers/local_http.rs entry. Signed-off-by: Jussi Elo --- apps/rocm/src/providers.rs | 283 +---------------- apps/rocm/src/providers/local_http.rs | 299 ++++++++++++++++++ .../providers/{ => local_http}/proptests.rs | 3 +- docs/architecture.md | 3 +- 4 files changed, 309 insertions(+), 279 deletions(-) create mode 100644 apps/rocm/src/providers/local_http.rs rename apps/rocm/src/providers/{ => local_http}/proptests.rs (99%) diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index d271d2d114..b56b03d7ab 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -2,18 +2,20 @@ // // SPDX-License-Identifier: MIT +mod local_http; mod tool_schema; +#[cfg(test)] +use self::local_http::LOCAL_STREAM_TRUNCATED; +use self::local_http::{post_json_to_local_endpoint, stream_json_from_local_endpoint}; use self::tool_schema::{rocm_anthropic_tool_definitions, rocm_openai_tool_definitions}; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AuditEventRecord, EndpointReadiness, EndpointReadinessOutcome, ManagedServiceRecord, - RocmCliConfig, append_audit_event, connect_tcp_stream, format_host_port, - managed_service_endpoint_readiness, read_http_response_bounded, unix_time_millis, - write_all_tcp_stream, + RocmCliConfig, append_audit_event, managed_service_endpoint_readiness, unix_time_millis, }; use std::fs; -use std::io::{BufRead, BufReader, Read}; +use std::io::Read; use std::time::Duration; pub(crate) const ROCM_TOOL_SCHEMA_ID: &str = "rocm-tools-v0"; @@ -560,268 +562,6 @@ fn ready_local_services(paths: &AppPaths) -> Result> { Ok(services) } -fn post_json_to_local_endpoint( - endpoint_url: &str, - path: &str, - body: &serde_json::Value, - endpoint_api_key: Option<&str>, - timeout: Duration, -) -> Result { - let body = - post_json_to_local_endpoint_body(endpoint_url, path, body, endpoint_api_key, timeout)?; - serde_json::from_str(body.trim()).context("failed to parse local provider chat JSON") -} - -/// `Authorization: Bearer` request-header line for a protected local endpoint, or -/// an empty string for an unauthenticated (loopback) one. -fn local_bearer_header(endpoint_api_key: Option<&str>) -> String { - match endpoint_api_key { - Some(key) => format!("Authorization: Bearer {key}\r\n"), - None => String::new(), - } -} - -fn post_json_to_local_endpoint_body( - endpoint_url: &str, - path: &str, - body: &serde_json::Value, - endpoint_api_key: Option<&str>, - timeout: Duration, -) -> Result { - let (host, port) = parse_http_endpoint(endpoint_url) - .with_context(|| format!("unsupported local endpoint URL `{endpoint_url}`"))?; - let body = serde_json::to_string(body).context("failed to serialize chat request")?; - let deadline = std::time::Instant::now() + timeout; - let mut stream = connect_tcp_stream(&host, port, timeout)?; - let host_header = format_host_port(&host, port); - let auth_header = local_bearer_header(endpoint_api_key); - let request = format!( - "POST {path} HTTP/1.1\r\nHost: {host_header}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n{auth_header}Connection: close\r\n\r\n{}", - body.len(), - body - ); - write_all_tcp_stream(&mut stream, request.as_bytes()) - .context("failed to write local provider chat request")?; - - let response = read_http_response_bounded(&mut stream, deadline) - .context("failed to read local provider chat response")?; - let (headers, body) = response - .split_once("\r\n\r\n") - .context("local provider response was missing HTTP body")?; - let status_line = headers.lines().next().unwrap_or_default(); - if !status_line.contains(" 200 ") { - bail!("local provider returned {status_line}"); - } - Ok(body.to_owned()) -} - -fn stream_json_from_local_endpoint( - endpoint_url: &str, - path: &str, - body: &serde_json::Value, - endpoint_api_key: Option<&str>, - timeout: Duration, - on_event: &mut dyn FnMut(ProviderStreamEvent) -> Result<()>, -) -> Result<()> { - let (host, port) = parse_http_endpoint(endpoint_url) - .with_context(|| format!("unsupported local endpoint URL `{endpoint_url}`"))?; - let body = serde_json::to_string(body).context("failed to serialize chat request")?; - let mut stream = connect_tcp_stream(&host, port, timeout)?; - let host_header = format_host_port(&host, port); - let auth_header = local_bearer_header(endpoint_api_key); - let request = format!( - "POST {path} HTTP/1.1\r\nHost: {host_header}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n{auth_header}Connection: close\r\n\r\n{}", - body.len(), - body - ); - write_all_tcp_stream(&mut stream, request.as_bytes()) - .context("failed to write local provider chat request")?; - - let mut reader = BufReader::new(stream); - let mut headers = String::new(); - loop { - let mut line = String::new(); - let read = reader - .read_line(&mut line) - .context("failed to read local provider response headers")?; - if read == 0 { - bail!("local provider response ended before headers completed"); - } - headers.push_str(&line); - if line == "\r\n" || line == "\n" { - break; - } - } - let status_line = headers.lines().next().unwrap_or_default(); - if !status_line.contains(" 200 ") { - bail!("local provider returned {status_line}"); - } - - let mut emitter = OpenAiSseEmitter::new(on_event); - read_sse_body(&mut reader, &headers, &mut emitter)?; - emitter.finish() -} - -/// Read a streamed response body in whichever framing `headers` declare: -/// chunked, `Content-Length`, or delimited by the connection closing. -fn read_sse_body( - reader: &mut R, - headers: &str, - emitter: &mut OpenAiSseEmitter<'_>, -) -> Result<()> { - let chunked = header_value(headers, "Transfer-Encoding") - .is_some_and(|value| value.to_ascii_lowercase().contains("chunked")); - let content_length = header_value(headers, "Content-Length") - .and_then(|value| value.trim().parse::().ok()); - if chunked { - read_chunked_sse_body(reader, emitter) - } else if let Some(length) = content_length { - read_fixed_sse_body(reader, length, emitter).map_err(body_truncated_at_eof) - } else { - // For this framing the connection closing *is* the end of the body, so - // end-of-stream is success here and the transport cannot tell a - // finished answer from one cut off mid-way. That asymmetry with the - // two framings above is deliberate: the only signal left is whether - // `data: [DONE]` arrived, and `OpenAiSseEmitter::finish` does not - // require it for any framing today. Making it required is a protocol - // policy for every engine, not a transport fix. - read_close_delimited_sse_body(reader, emitter) - } -} - -fn header_value(headers: &str, name: &str) -> Option { - headers.lines().find_map(|line| { - let (key, value) = line.split_once(':')?; - key.eq_ignore_ascii_case(name) - .then(|| value.trim().to_owned()) - }) -} - -fn read_fixed_sse_body( - reader: &mut R, - mut remaining: usize, - emitter: &mut OpenAiSseEmitter<'_>, -) -> Result<()> { - let mut buffer = [0_u8; 1024]; - while remaining > 0 { - let read_len = remaining.min(buffer.len()); - reader - .read_exact(&mut buffer[..read_len]) - .context("failed to read local provider SSE body")?; - emitter.push_bytes(&buffer[..read_len])?; - remaining -= read_len; - } - Ok(()) -} - -fn read_close_delimited_sse_body( - reader: &mut R, - emitter: &mut OpenAiSseEmitter<'_>, -) -> Result<()> { - let mut buffer = [0_u8; 1024]; - loop { - let read = match reader.read(&mut buffer) { - Ok(0) => return Ok(()), - Ok(read) => read, - // A signal delivered to this thread aborts the socket read with - // `EINTR`, and Linux never restarts a read that has a receive - // timeout set (see signal(7)), which this socket has. Nothing is - // wrong with the connection, so read again. (`read_line` and - // `read_exact`, used by the other body framings, retry internally; - // a bare `read` does not.) - Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, - Err(error) => return Err(error).context("failed to read local provider SSE body"), - }; - emitter.push_bytes(&buffer[..read])?; - } -} - -const LOCAL_STREAM_TRUNCATED: &str = "local provider closed the connection before the streamed response finished; \ - the response is incomplete"; - -/// Report an end-of-stream before a body's declared end — inside a chunk (its -/// size line, data or CRLF) or short of its `Content-Length` — the same way as -/// one between chunks: either way the engine went away mid-answer. Any other -/// error (a read timeout, a reset connection, a malformed line) passes through -/// unchanged, so it is not mislabelled as the engine closing the connection. -fn body_truncated_at_eof(error: anyhow::Error) -> anyhow::Error { - let at_eof = error.chain().any(|cause| { - cause - .downcast_ref::() - .is_some_and(|io| io.kind() == std::io::ErrorKind::UnexpectedEof) - }); - if at_eof { - error.context(LOCAL_STREAM_TRUNCATED) - } else { - error - } -} - -fn read_chunked_sse_body( - reader: &mut R, - emitter: &mut OpenAiSseEmitter<'_>, -) -> Result<()> { - loop { - let mut size_line = String::new(); - let read = reader - .read_line(&mut size_line) - .context("failed to read local provider chunk size")?; - // `Ok(0)` is end-of-stream, not a blank line: the engine closed the - // connection (it crashed, was killed, or was stopped) before sending the - // terminating zero-length chunk. Reading again would return `Ok(0)` - // immediately, forever. - if read == 0 { - bail!("{LOCAL_STREAM_TRUNCATED}"); - } - if size_line.trim().is_empty() { - continue; - } - let size_text = size_line - .trim() - .split(';') - .next() - .unwrap_or_default() - .trim(); - let size = usize::from_str_radix(size_text, 16) - .with_context(|| format!("invalid local provider chunk size `{size_text}`"))?; - if size == 0 { - // The zero chunk's own CRLF (or a trailer line). Its read result is - // deliberately ignored: every event, `[DONE]` included, has been - // delivered by now, so an engine that closes or stalls here has - // still sent a complete answer, and failing would discard it. - let mut trailer = String::new(); - let _ = reader.read_line(&mut trailer); - break; - } - read_fixed_sse_body(reader, size, emitter).map_err(body_truncated_at_eof)?; - let mut crlf = [0_u8; 2]; - reader - .read_exact(&mut crlf) - .context("failed to read local provider chunk terminator") - .map_err(body_truncated_at_eof)?; - if crlf != *b"\r\n" { - bail!("invalid local provider chunk terminator"); - } - } - Ok(()) -} - -fn parse_http_endpoint(endpoint_url: &str) -> Option<(String, u16)> { - let without_scheme = endpoint_url.trim().strip_prefix("http://")?; - let authority = without_scheme.split('/').next()?.trim(); - if authority.is_empty() { - return None; - } - if let Some(rest) = authority.strip_prefix('[') { - let end = rest.find(']')?; - let host = rest[..end].to_owned(); - let port = rest[end + 1..].strip_prefix(':')?.parse().ok()?; - return Some((host, port)); - } - let (host, port) = authority.rsplit_once(':')?; - Some((host.to_owned(), port.parse().ok()?)) -} - #[cfg(test)] fn parse_openai_chat_content(response: &serde_json::Value) -> Result { let (content, tool_calls) = parse_openai_chat_message(response)?; @@ -1427,9 +1167,6 @@ fn parse_anthropic_sse_line(line: &str) -> Result> { } } -#[cfg(test)] -mod proptests; - #[cfg(test)] mod tests { use super::*; @@ -1448,14 +1185,6 @@ mod tests { assert_eq!(BUILTIN_ASSISTANT_MODEL_ID, "Qwen3-4B-Instruct-2507-GGUF"); } - #[test] - fn local_provider_endpoint_parser_supports_ipv6_loopback() { - assert_eq!( - parse_http_endpoint("http://[::1]:11435/v1"), - Some(("::1".to_owned(), 11435)) - ); - } - #[test] fn local_provider_reports_ready_managed_models() -> Result<()> { let (root, paths) = temp_app_paths("local-provider-models"); diff --git a/apps/rocm/src/providers/local_http.rs b/apps/rocm/src/providers/local_http.rs new file mode 100644 index 0000000000..ce87ea5e8e --- /dev/null +++ b/apps/rocm/src/providers/local_http.rs @@ -0,0 +1,299 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The local provider's hand-rolled HTTP/1.1 transport: a blocking JSON POST +//! ([`post_json_to_local_endpoint`]) and a streamed POST that feeds an SSE +//! emitter in whichever body framing the engine answers with +//! ([`stream_json_from_local_endpoint`]: chunked, `Content-Length`, or +//! close-delimited), plus the `http://` endpoint parser both rely on. +//! +//! The streamed path decodes into the OpenAI-format emitter, so this module +//! reads `OpenAiSseEmitter`; that is the only edge back into `providers` +//! besides its shared types. + +use super::{OpenAiSseEmitter, ProviderStreamEvent, SseBytesEmitter}; +use anyhow::{Context, Result, bail}; +use rocm_core::{ + connect_tcp_stream, format_host_port, read_http_response_bounded, write_all_tcp_stream, +}; +use std::io::{BufRead, BufReader, Read}; +use std::time::Duration; + +pub(super) fn post_json_to_local_endpoint( + endpoint_url: &str, + path: &str, + body: &serde_json::Value, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + let body = + post_json_to_local_endpoint_body(endpoint_url, path, body, endpoint_api_key, timeout)?; + serde_json::from_str(body.trim()).context("failed to parse local provider chat JSON") +} + +/// `Authorization: Bearer` request-header line for a protected local endpoint, or +/// an empty string for an unauthenticated (loopback) one. +fn local_bearer_header(endpoint_api_key: Option<&str>) -> String { + match endpoint_api_key { + Some(key) => format!("Authorization: Bearer {key}\r\n"), + None => String::new(), + } +} + +fn post_json_to_local_endpoint_body( + endpoint_url: &str, + path: &str, + body: &serde_json::Value, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + let (host, port) = parse_http_endpoint(endpoint_url) + .with_context(|| format!("unsupported local endpoint URL `{endpoint_url}`"))?; + let body = serde_json::to_string(body).context("failed to serialize chat request")?; + let deadline = std::time::Instant::now() + timeout; + let mut stream = connect_tcp_stream(&host, port, timeout)?; + let host_header = format_host_port(&host, port); + let auth_header = local_bearer_header(endpoint_api_key); + let request = format!( + "POST {path} HTTP/1.1\r\nHost: {host_header}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n{auth_header}Connection: close\r\n\r\n{}", + body.len(), + body + ); + write_all_tcp_stream(&mut stream, request.as_bytes()) + .context("failed to write local provider chat request")?; + + let response = read_http_response_bounded(&mut stream, deadline) + .context("failed to read local provider chat response")?; + let (headers, body) = response + .split_once("\r\n\r\n") + .context("local provider response was missing HTTP body")?; + let status_line = headers.lines().next().unwrap_or_default(); + if !status_line.contains(" 200 ") { + bail!("local provider returned {status_line}"); + } + Ok(body.to_owned()) +} + +pub(super) fn stream_json_from_local_endpoint( + endpoint_url: &str, + path: &str, + body: &serde_json::Value, + endpoint_api_key: Option<&str>, + timeout: Duration, + on_event: &mut dyn FnMut(ProviderStreamEvent) -> Result<()>, +) -> Result<()> { + let (host, port) = parse_http_endpoint(endpoint_url) + .with_context(|| format!("unsupported local endpoint URL `{endpoint_url}`"))?; + let body = serde_json::to_string(body).context("failed to serialize chat request")?; + let mut stream = connect_tcp_stream(&host, port, timeout)?; + let host_header = format_host_port(&host, port); + let auth_header = local_bearer_header(endpoint_api_key); + let request = format!( + "POST {path} HTTP/1.1\r\nHost: {host_header}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n{auth_header}Connection: close\r\n\r\n{}", + body.len(), + body + ); + write_all_tcp_stream(&mut stream, request.as_bytes()) + .context("failed to write local provider chat request")?; + + let mut reader = BufReader::new(stream); + let mut headers = String::new(); + loop { + let mut line = String::new(); + let read = reader + .read_line(&mut line) + .context("failed to read local provider response headers")?; + if read == 0 { + bail!("local provider response ended before headers completed"); + } + headers.push_str(&line); + if line == "\r\n" || line == "\n" { + break; + } + } + let status_line = headers.lines().next().unwrap_or_default(); + if !status_line.contains(" 200 ") { + bail!("local provider returned {status_line}"); + } + + let mut emitter = OpenAiSseEmitter::new(on_event); + read_sse_body(&mut reader, &headers, &mut emitter)?; + emitter.finish() +} + +/// Read a streamed response body in whichever framing `headers` declare: +/// chunked, `Content-Length`, or delimited by the connection closing. +fn read_sse_body( + reader: &mut R, + headers: &str, + emitter: &mut OpenAiSseEmitter<'_>, +) -> Result<()> { + let chunked = header_value(headers, "Transfer-Encoding") + .is_some_and(|value| value.to_ascii_lowercase().contains("chunked")); + let content_length = header_value(headers, "Content-Length") + .and_then(|value| value.trim().parse::().ok()); + if chunked { + read_chunked_sse_body(reader, emitter) + } else if let Some(length) = content_length { + read_fixed_sse_body(reader, length, emitter).map_err(body_truncated_at_eof) + } else { + // For this framing the connection closing *is* the end of the body, so + // end-of-stream is success here and the transport cannot tell a + // finished answer from one cut off mid-way. That asymmetry with the + // two framings above is deliberate: the only signal left is whether + // `data: [DONE]` arrived, and `OpenAiSseEmitter::finish` does not + // require it for any framing today. Making it required is a protocol + // policy for every engine, not a transport fix. + read_close_delimited_sse_body(reader, emitter) + } +} + +fn header_value(headers: &str, name: &str) -> Option { + headers.lines().find_map(|line| { + let (key, value) = line.split_once(':')?; + key.eq_ignore_ascii_case(name) + .then(|| value.trim().to_owned()) + }) +} + +fn read_fixed_sse_body( + reader: &mut R, + mut remaining: usize, + emitter: &mut OpenAiSseEmitter<'_>, +) -> Result<()> { + let mut buffer = [0_u8; 1024]; + while remaining > 0 { + let read_len = remaining.min(buffer.len()); + reader + .read_exact(&mut buffer[..read_len]) + .context("failed to read local provider SSE body")?; + emitter.push_bytes(&buffer[..read_len])?; + remaining -= read_len; + } + Ok(()) +} + +fn read_close_delimited_sse_body( + reader: &mut R, + emitter: &mut OpenAiSseEmitter<'_>, +) -> Result<()> { + let mut buffer = [0_u8; 1024]; + loop { + let read = match reader.read(&mut buffer) { + Ok(0) => return Ok(()), + Ok(read) => read, + // A signal delivered to this thread aborts the socket read with + // `EINTR`, and Linux never restarts a read that has a receive + // timeout set (see signal(7)), which this socket has. Nothing is + // wrong with the connection, so read again. (`read_line` and + // `read_exact`, used by the other body framings, retry internally; + // a bare `read` does not.) + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error).context("failed to read local provider SSE body"), + }; + emitter.push_bytes(&buffer[..read])?; + } +} + +pub(super) const LOCAL_STREAM_TRUNCATED: &str = "local provider closed the connection before the streamed response finished; \ + the response is incomplete"; + +/// Report an end-of-stream before a body's declared end — inside a chunk (its +/// size line, data or CRLF) or short of its `Content-Length` — the same way as +/// one between chunks: either way the engine went away mid-answer. Any other +/// error (a read timeout, a reset connection, a malformed line) passes through +/// unchanged, so it is not mislabelled as the engine closing the connection. +fn body_truncated_at_eof(error: anyhow::Error) -> anyhow::Error { + let at_eof = error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::UnexpectedEof) + }); + if at_eof { + error.context(LOCAL_STREAM_TRUNCATED) + } else { + error + } +} + +fn read_chunked_sse_body( + reader: &mut R, + emitter: &mut OpenAiSseEmitter<'_>, +) -> Result<()> { + loop { + let mut size_line = String::new(); + let read = reader + .read_line(&mut size_line) + .context("failed to read local provider chunk size")?; + // `Ok(0)` is end-of-stream, not a blank line: the engine closed the + // connection (it crashed, was killed, or was stopped) before sending the + // terminating zero-length chunk. Reading again would return `Ok(0)` + // immediately, forever. + if read == 0 { + bail!("{LOCAL_STREAM_TRUNCATED}"); + } + if size_line.trim().is_empty() { + continue; + } + let size_text = size_line + .trim() + .split(';') + .next() + .unwrap_or_default() + .trim(); + let size = usize::from_str_radix(size_text, 16) + .with_context(|| format!("invalid local provider chunk size `{size_text}`"))?; + if size == 0 { + // The zero chunk's own CRLF (or a trailer line). Its read result is + // deliberately ignored: every event, `[DONE]` included, has been + // delivered by now, so an engine that closes or stalls here has + // still sent a complete answer, and failing would discard it. + let mut trailer = String::new(); + let _ = reader.read_line(&mut trailer); + break; + } + read_fixed_sse_body(reader, size, emitter).map_err(body_truncated_at_eof)?; + let mut crlf = [0_u8; 2]; + reader + .read_exact(&mut crlf) + .context("failed to read local provider chunk terminator") + .map_err(body_truncated_at_eof)?; + if crlf != *b"\r\n" { + bail!("invalid local provider chunk terminator"); + } + } + Ok(()) +} + +fn parse_http_endpoint(endpoint_url: &str) -> Option<(String, u16)> { + let without_scheme = endpoint_url.trim().strip_prefix("http://")?; + let authority = without_scheme.split('/').next()?.trim(); + if authority.is_empty() { + return None; + } + if let Some(rest) = authority.strip_prefix('[') { + let end = rest.find(']')?; + let host = rest[..end].to_owned(); + let port = rest[end + 1..].strip_prefix(':')?.parse().ok()?; + return Some((host, port)); + } + let (host, port) = authority.rsplit_once(':')?; + Some((host.to_owned(), port.parse().ok()?)) +} + +#[cfg(test)] +mod proptests; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn local_provider_endpoint_parser_supports_ipv6_loopback() { + assert_eq!( + parse_http_endpoint("http://[::1]:11435/v1"), + Some(("::1".to_owned(), 11435)) + ); + } +} diff --git a/apps/rocm/src/providers/proptests.rs b/apps/rocm/src/providers/local_http/proptests.rs similarity index 99% rename from apps/rocm/src/providers/proptests.rs rename to apps/rocm/src/providers/local_http/proptests.rs index f576ed3ae4..af0951f30a 100644 --- a/apps/rocm/src/providers/proptests.rs +++ b/apps/rocm/src/providers/local_http/proptests.rs @@ -2,7 +2,8 @@ // // SPDX-License-Identifier: MIT -//! Property tests for the local provider's SSE transport readers. +//! Property tests for the local provider's SSE transport readers, declared +//! from `local_http.rs` so they sit with the readers they test. //! //! Each property re-frames one of the SSE bodies the unit tests in //! `providers.rs` already use (plus the same shape with CRLF line endings and diff --git a/docs/architecture.md b/docs/architecture.md index 68b2d43ab4..3bbb0a9c06 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -61,8 +61,9 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat.rs`](../apps/rocm/src/chat.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat.rs`](../apps/rocm/src/chat.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. - [`model_fit.rs`](../apps/rocm/src/model_fit.rs) — mechanical relocation, owns a `pub(crate)` type: answering whether a host can run a model for `rocm diagnose --model` (`assess_model_on_this_host`, `assess_model_with_host_paths`, `assess_model_for_host`), the accelerator-memory fact it rests on (`host_accelerator_memory`, `classify_accelerator_memory`, `examination_reports_amd_gpu`) and the platform gate that rules an engine out (`PlatformRuleOut`, `engine_ruled_out_by_platform`, `unsupported_here_for`). It is a module of its own, rather than part of the `diagnose` handler, so [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs), which reads `PlatformRuleOut` and `engine_ruled_out_by_platform`, does not depend on a command module. It depends on [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`select_serve_engine`) and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs), neither of which depends on it. `assess_model_on_this_host` (called from [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` are `pub(crate)`, and [`main.rs`](../apps/rocm/src/main.rs) re-imports the last two so `examine_report_text.rs` reaches them as `crate::`; every other item is private. The cluster's regression tests live in this module's own `mod tests`; the one that re-runs the test binary on a helper test by its full path (`engine_ruled_out_by_platform_gates_aim_on_docker_reachability`) names the helper's new path, `model_fit::tests::docker_unreachable_subprocess_helper`. - [`provider_keys.rs`](../apps/rocm/src/provider_keys.rs) — full domain extraction. -- [`providers.rs`](../apps/rocm/src/providers.rs) — full domain extraction (no dedicated command enum; invoked from the existing chat/config command flows). Keeps the request/response types, the `ProviderAdapter` trait and the `provider_*` dispatch, the `LocalProvider` and `RemoteProvider` implementations, local-service selection (`select_local_chat_service`, `ready_local_services`), the remote model and endpoint resolution, the `ureq` remote transport and the `SseBytesEmitter` trait. The ROCm tool schema lives in a child module under `providers/`, below. +- [`providers.rs`](../apps/rocm/src/providers.rs) — full domain extraction (no dedicated command enum; invoked from the existing chat/config command flows). Keeps the request/response types, the `ProviderAdapter` trait and the `provider_*` dispatch, the `LocalProvider` and `RemoteProvider` implementations, local-service selection (`select_local_chat_service`, `ready_local_services`), the remote model and endpoint resolution, the `ureq` remote transport and the `SseBytesEmitter` trait. The ROCm tool schema and the local HTTP/SSE transport live in child modules under `providers/`, below. - [`providers/tool_schema.rs`](../apps/rocm/src/providers/tool_schema.rs) — full domain extraction, a leaf child of `providers.rs`: the ROCm tool catalog a provider advertises to a model, as OpenAI function definitions (`rocm_openai_tool_definitions`) and reshaped for Anthropic's `tools` field (`rocm_anthropic_tool_definitions`). Both are `pub(super)`, read by the request-body builders in `providers.rs`; `rocm_openai_tool`, the one-tool constructor behind them, is private. Depends on nothing else in `providers`. +- [`providers/local_http.rs`](../apps/rocm/src/providers/local_http.rs) — full domain extraction, a child of `providers.rs`: the local provider's hand-rolled HTTP/1.1 transport. `post_json_to_local_endpoint` (with `post_json_to_local_endpoint_body` and `local_bearer_header`) is the blocking JSON POST; `stream_json_from_local_endpoint` is the streamed POST, which reads the response in whichever body framing the engine answers with (`read_sse_body` picks between `read_chunked_sse_body`, `read_fixed_sse_body` and `read_close_delimited_sse_body` from the headers, via `header_value`) and feeds an `OpenAiSseEmitter`; `body_truncated_at_eof` and `LOCAL_STREAM_TRUNCATED` name the failure when an engine closes the connection mid-answer; `parse_http_endpoint` reads the `http://` endpoint URL. `post_json_to_local_endpoint` and `stream_json_from_local_endpoint` are `pub(super)`, called from `LocalProvider`, and `LOCAL_STREAM_TRUNCATED` is also `pub(super)` (imported by `providers.rs` for its test module only); the rest is private. Its edge back into `providers` is the emitter and the `SseBytesEmitter` trait it pushes bytes through, plus the shared `ProviderStreamEvent` type. [`providers/local_http/proptests.rs`](../apps/rocm/src/providers/local_http/proptests.rs) holds the property tests of the SSE body readers and is declared from this module so they sit with the readers they test; the example-based test of `parse_http_endpoint` is in this module's own `mod tests`, and the tests that drive `LocalProvider` end to end against a mock server stay in `providers.rs`'s `mod tests` with their server helpers. - [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. - [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install: the orchestration (`finish_sdk_install`, `settle_engine_install`, `finalize_successful_sdk_install`), the engine-dependency check, the runtime device check and the post-install verdict built on it, and `SdkInstallConsents`. The torch alignment and retention it runs live in the child module [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs). `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`; `settle_engine_install` is `pub(crate)` and read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs). The host system-package installs live in [`system_packages.rs`](../apps/rocm/src/system_packages.rs); this module re-exports `ensure_libatomic_for_torch`/`ensure_libnuma_for_torch` (called from both `install_cmd.rs`'s `install()` and `engines_cmd.rs`) and `ensure_openmpi_for_vllm` (called from `engines_cmd.rs` and, for the engine auto-install after an SDK install, from here) so those callers keep their imports. - [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) — full domain extraction, a child of `runtime_install_finalize.rs` that depends on its parent's device check (`RuntimeDeviceCheck`, `device_check_verdict`) and on `runtime_manifest_for_selector` in [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): what to do about the torch an engine install finds in a managed runtime. `plan_torch_alignment` decides whether the SDK's torch must be realigned to the engine's pin, `align_runtime_torch` runs the realignment, `classify_retained_torch` decides when a torch that already works is kept as an intended divergence instead, and the `render_*` and `report_*` functions with `TorchAlignment` and `TorchRetention` print and log the outcome. The manifest readers (`sdk_torch_build_for_key`, `sdk_torch_build_from_manifest`, `runtime_python_for_key`, `runtime_library_paths_for_key`, `runtime_index_url_for_key`) live here too. The parent stays the caller (`settle_engine_install` and `maybe_auto_install_sdk_preferred_engine`), so it could not be a crate-level sibling without a cycle with the parent's device check. The items the parent calls are `pub(super)`: `TorchAlignment`, `TorchRetention`, `report_torch_alignment`, `report_torch_retention`, `classify_retained_torch`, `retained_diverged_package`, `deliberately_diverged_package`, `sdk_torch_build_for_key`, `runtime_python_for_key` and `runtime_library_paths_for_key`; the rest is private. Its tests cover planning, rendering, the opt-out and the retention rules, and reach the parent's private engine-dependency check as a descendant of it. From 67a2c9f43a8288144aed1965a73e81be29aca2d8 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 15:28:27 +0000 Subject: [PATCH 3/4] refactor(providers): move the provider wire formats into providers/wire.rs The OpenAI-compatible and Anthropic request bodies, the parsers for their responses and the SSE emitters that decode a streamed answer were interleaved with provider dispatch in providers.rs. Move them into a child module, providers/wire.rs: the openai_*_request_body* and anthropic_*_request_body* builders, apply_local_model_request_defaults, local_model_should_disable_thinking, local_openai_compatible_request, the parse_openai_* and parse_anthropic_* parsers (including the SSE line parsers and the test-only content helpers), and OpenAiSseEmitter and AnthropicSseEmitter with their SseBytesEmitter impls. The SseBytesEmitter trait stays in providers.rs, with the ureq remote transport. This is a pure move: the bodies are byte-identical apart from rustfmt re-wrapping signatures that gained pub(super). The builders, the two chat-message parsers and the two emitters (with new and finish) become pub(super); everything else stays private. local_http now reaches OpenAiSseEmitter through wire, and wire reaches only the shared types in providers and tool_schema, so the dependency between them is one-way. The ten tests of the builders and parsers move into wire.rs's own test module with their leaf names unchanged, and the emitter property tests (sse_emitter_proptests.rs) are declared from wire.rs so they sit with the emitters and line parsers they test. The tests that drive LocalProvider or the remote providers against a mock server stay in providers.rs, with spawn_local_provider_test_server and spawn_sequenced_local_provider_test_server. docs/architecture.md gains the providers/wire.rs entry. Signed-off-by: Jussi Elo --- apps/rocm/src/providers.rs | 828 +---------------- apps/rocm/src/providers/local_http.rs | 3 +- .../src/providers/local_http/proptests.rs | 2 +- apps/rocm/src/providers/wire.rs | 855 ++++++++++++++++++ .../{ => wire}/sse_emitter_proptests.rs | 6 +- docs/architecture.md | 7 +- 6 files changed, 872 insertions(+), 829 deletions(-) create mode 100644 apps/rocm/src/providers/wire.rs rename apps/rocm/src/providers/{ => wire}/sse_emitter_proptests.rs (99%) diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index b56b03d7ab..87d6c9ff44 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -4,11 +4,17 @@ mod local_http; mod tool_schema; +mod wire; #[cfg(test)] use self::local_http::LOCAL_STREAM_TRUNCATED; use self::local_http::{post_json_to_local_endpoint, stream_json_from_local_endpoint}; -use self::tool_schema::{rocm_anthropic_tool_definitions, rocm_openai_tool_definitions}; +use self::wire::{ + AnthropicSseEmitter, OpenAiSseEmitter, anthropic_chat_request_body, + anthropic_stream_chat_request_body, openai_chat_request_body, openai_local_chat_request_body, + openai_local_stream_chat_request_body, openai_stream_chat_request_body, + parse_anthropic_chat_message, parse_openai_chat_message, +}; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AuditEventRecord, EndpointReadiness, EndpointReadinessOutcome, ManagedServiceRecord, @@ -562,125 +568,6 @@ fn ready_local_services(paths: &AppPaths) -> Result> { Ok(services) } -#[cfg(test)] -fn parse_openai_chat_content(response: &serde_json::Value) -> Result { - let (content, tool_calls) = parse_openai_chat_message(response)?; - if content.is_empty() && !tool_calls.is_empty() { - bail!("OpenAI-compatible response contained tool calls but no text content"); - } - Ok(content) -} - -fn parse_openai_chat_message(response: &serde_json::Value) -> Result<(String, Vec)> { - let message = response - .get("choices") - .and_then(serde_json::Value::as_array) - .and_then(|choices| choices.first()) - .and_then(|choice| choice.get("message")) - .context("OpenAI-compatible response did not include choices[0].message")?; - let content = message - .get("content") - .and_then(serde_json::Value::as_str) - .map(str::to_owned) - .unwrap_or_default(); - let tool_calls = parse_openai_tool_calls(message)?; - if content.is_empty() && tool_calls.is_empty() { - bail!("OpenAI-compatible response did not include message content or tool calls"); - } - Ok((content, tool_calls)) -} - -fn parse_openai_tool_calls(message: &serde_json::Value) -> Result> { - let Some(items) = message - .get("tool_calls") - .and_then(serde_json::Value::as_array) - else { - return Ok(Vec::new()); - }; - let mut calls = Vec::new(); - for item in items { - let function = item - .get("function") - .context("OpenAI-compatible tool call was missing function")?; - let name = function - .get("name") - .and_then(serde_json::Value::as_str) - .map(str::to_owned) - .context("OpenAI-compatible tool call was missing function.name")?; - let arguments = match function.get("arguments") { - Some(serde_json::Value::String(text)) if text.trim().is_empty() => { - serde_json::json!({}) - } - Some(serde_json::Value::String(text)) => serde_json::from_str(text) - .with_context(|| format!("tool call `{name}` arguments were not valid JSON"))?, - Some(value) => value.clone(), - None => serde_json::json!({}), - }; - calls.push(ChatToolCall { - id: item - .get("id") - .and_then(serde_json::Value::as_str) - .map(str::to_owned), - name, - arguments, - }); - } - Ok(calls) -} - -#[cfg(test)] -fn parse_anthropic_chat_content(response: &serde_json::Value) -> Result { - let (content, tool_calls) = parse_anthropic_chat_message(response)?; - if content.is_empty() && !tool_calls.is_empty() { - bail!("anthropic provider response contained tool calls but no text content"); - } - Ok(content) -} - -fn parse_anthropic_chat_message( - response: &serde_json::Value, -) -> Result<(String, Vec)> { - let items = response - .get("content") - .and_then(serde_json::Value::as_array) - .context("anthropic provider response did not include content blocks")?; - let mut text = String::new(); - let mut tool_calls = Vec::new(); - for item in items { - match item.get("type").and_then(serde_json::Value::as_str) { - Some("text") => { - if let Some(chunk) = item.get("text").and_then(serde_json::Value::as_str) { - text.push_str(chunk); - } - } - Some("tool_use") => { - let name = item - .get("name") - .and_then(serde_json::Value::as_str) - .map(str::to_owned) - .context("anthropic provider tool_use block was missing name")?; - let arguments = item - .get("input") - .cloned() - .unwrap_or_else(|| serde_json::json!({})); - tool_calls.push(ChatToolCall { - id: item - .get("id") - .and_then(serde_json::Value::as_str) - .map(str::to_owned), - name, - arguments, - }); - } - _ => {} - } - } - if text.is_empty() && tool_calls.is_empty() { - bail!("anthropic provider response did not include text content or tool calls"); - } - Ok((text, tool_calls)) -} - fn resolve_remote_model( provider: &str, model_env: &str, @@ -707,158 +594,6 @@ fn remote_endpoint(endpoint_env: &str, default_endpoint: &str) -> String { .unwrap_or_else(|| default_endpoint.to_owned()) } -fn openai_chat_request_body(model: &str, request: &ChatRequest) -> serde_json::Value { - openai_chat_request_body_with_stream(model, request, false) -} - -fn openai_stream_chat_request_body(model: &str, request: &ChatRequest) -> serde_json::Value { - openai_chat_request_body_with_stream(model, request, true) -} - -fn openai_local_chat_request_body(model: &str, request: &ChatRequest) -> serde_json::Value { - let request = local_openai_compatible_request(request); - let mut body = openai_chat_request_body(model, &request); - apply_local_model_request_defaults(model, &mut body); - body -} - -fn openai_local_stream_chat_request_body(model: &str, request: &ChatRequest) -> serde_json::Value { - let request = local_openai_compatible_request(request); - let mut body = openai_stream_chat_request_body(model, &request); - apply_local_model_request_defaults(model, &mut body); - body -} - -fn apply_local_model_request_defaults(model: &str, body: &mut serde_json::Value) { - if local_model_should_disable_thinking(model) { - body["chat_template_kwargs"] = serde_json::json!({ - "enable_thinking": false - }); - } -} - -fn local_model_should_disable_thinking(model: &str) -> bool { - let normalized = model.to_ascii_lowercase(); - normalized.contains("qwen3") || normalized.contains("qwen/qwen3") -} - -fn local_openai_compatible_request(request: &ChatRequest) -> ChatRequest { - let system = request - .messages - .iter() - .filter(|message| message.role == "system") - .map(|message| message.content.trim()) - .filter(|content| !content.is_empty()) - .collect::>() - .join("\n\n"); - let mut messages = request - .messages - .iter() - .filter(|message| message.role != "system") - .cloned() - .collect::>(); - if !system.is_empty() { - if let Some(first_user) = messages.iter_mut().find(|message| message.role == "user") { - first_user.content = format!( - "ROCm CLI guidance:\n{system}\n\nUser message:\n{}", - first_user.content - ); - } else { - messages.insert( - 0, - ChatMessage { - role: "user".to_owned(), - content: format!("ROCm CLI guidance:\n{system}"), - }, - ); - } - } - ChatRequest { - model: request.model.clone(), - messages, - max_tokens: request.max_tokens, - temperature: request.temperature, - top_p: request.top_p, - rocm_tools: request.rocm_tools, - } -} - -fn openai_chat_request_body_with_stream( - model: &str, - request: &ChatRequest, - stream: bool, -) -> serde_json::Value { - let mut body = serde_json::json!({ - "model": model, - "messages": request.messages.iter().map(|message| { - serde_json::json!({ - "role": message.role, - "content": message.content, - }) - }).collect::>(), - "stream": stream, - "max_tokens": request.max_tokens.unwrap_or(512), - }); - if let Some(temperature) = request.temperature { - body["temperature"] = serde_json::json!(temperature); - } - if let Some(top_p) = request.top_p { - body["top_p"] = serde_json::json!(top_p); - } - if request.rocm_tools && !stream { - body["tools"] = serde_json::Value::Array(rocm_openai_tool_definitions()); - body["tool_choice"] = serde_json::json!("auto"); - } - body -} - -fn anthropic_chat_request_body(model: &str, request: &ChatRequest) -> serde_json::Value { - anthropic_chat_request_body_with_stream(model, request, false) -} - -fn anthropic_stream_chat_request_body(model: &str, request: &ChatRequest) -> serde_json::Value { - anthropic_chat_request_body_with_stream(model, request, true) -} - -fn anthropic_chat_request_body_with_stream( - model: &str, - request: &ChatRequest, - stream: bool, -) -> serde_json::Value { - let mut body = serde_json::json!({ - "model": model, - "messages": request.messages.iter().filter(|message| message.role != "system").map(|message| { - serde_json::json!({ - "role": if message.role == "assistant" { "assistant" } else { "user" }, - "content": message.content, - }) - }).collect::>(), - "max_tokens": request.max_tokens.unwrap_or(512), - "stream": stream, - }); - let system = request - .messages - .iter() - .filter(|message| message.role == "system") - .map(|message| message.content.trim()) - .filter(|content| !content.is_empty()) - .collect::>() - .join("\n\n"); - if !system.is_empty() { - body["system"] = serde_json::Value::String(system); - } - if let Some(temperature) = request.temperature { - body["temperature"] = serde_json::json!(temperature); - } - if let Some(top_p) = request.top_p { - body["top_p"] = serde_json::json!(top_p); - } - if request.rocm_tools && !stream { - body["tools"] = serde_json::Value::Array(rocm_anthropic_tool_definitions()); - } - body -} - fn post_json_with_headers( endpoint: &str, headers: &[(&str, String)], @@ -937,236 +672,6 @@ fn stream_json_with_headers( Ok(()) } -struct OpenAiSseEmitter<'a> { - pending: Vec, - done: bool, - on_event: &'a mut dyn FnMut(ProviderStreamEvent) -> Result<()>, -} - -impl<'a> OpenAiSseEmitter<'a> { - fn new(on_event: &'a mut dyn FnMut(ProviderStreamEvent) -> Result<()>) -> Self { - Self { - pending: Vec::new(), - done: false, - on_event, - } - } - - fn finish(mut self) -> Result<()> { - if !self.pending.is_empty() { - self.emit_pending_line()?; - } - if !self.done { - (self.on_event)(ProviderStreamEvent { - content: String::new(), - done: true, - })?; - self.done = true; - } - Ok(()) - } - - fn emit_pending_line(&mut self) -> Result<()> { - let line = String::from_utf8_lossy(&self.pending) - .trim_end_matches('\r') - .to_owned(); - self.pending.clear(); - self.emit_line(&line) - } - - fn emit_line(&mut self, line: &str) -> Result<()> { - if self.done { - return Ok(()); - } - let Some(event) = parse_openai_sse_line(line)? else { - return Ok(()); - }; - if event.done { - self.done = true; - } - (self.on_event)(event) - } -} - -impl SseBytesEmitter for OpenAiSseEmitter<'_> { - fn push_bytes(&mut self, bytes: &[u8]) -> Result<()> { - for byte in bytes { - if *byte == b'\n' { - self.emit_pending_line()?; - } else { - self.pending.push(*byte); - } - } - Ok(()) - } -} - -struct AnthropicSseEmitter<'a> { - pending: Vec, - done: bool, - on_event: &'a mut dyn FnMut(ProviderStreamEvent) -> Result<()>, -} - -impl<'a> AnthropicSseEmitter<'a> { - fn new(on_event: &'a mut dyn FnMut(ProviderStreamEvent) -> Result<()>) -> Self { - Self { - pending: Vec::new(), - done: false, - on_event, - } - } - - fn finish(mut self) -> Result<()> { - if !self.pending.is_empty() { - self.emit_pending_line()?; - } - if !self.done { - (self.on_event)(ProviderStreamEvent { - content: String::new(), - done: true, - })?; - self.done = true; - } - Ok(()) - } - - fn emit_pending_line(&mut self) -> Result<()> { - let line = String::from_utf8_lossy(&self.pending) - .trim_end_matches('\r') - .to_owned(); - self.pending.clear(); - self.emit_line(&line) - } - - fn emit_line(&mut self, line: &str) -> Result<()> { - if self.done { - return Ok(()); - } - let Some(event) = parse_anthropic_sse_line(line)? else { - return Ok(()); - }; - if event.done { - self.done = true; - } - (self.on_event)(event) - } -} - -impl SseBytesEmitter for AnthropicSseEmitter<'_> { - fn push_bytes(&mut self, bytes: &[u8]) -> Result<()> { - for byte in bytes { - if *byte == b'\n' { - self.emit_pending_line()?; - } else { - self.pending.push(*byte); - } - } - Ok(()) - } -} - -#[cfg(test)] -fn parse_openai_sse_chat_events(body: &str) -> Result> { - let mut events = Vec::new(); - for line in body.lines() { - let Some(event) = parse_openai_sse_line(line)? else { - continue; - }; - events.push(event); - } - if !events.iter().any(|event| event.done) { - events.push(ProviderStreamEvent { - content: String::new(), - done: true, - }); - } - Ok(events) -} - -fn parse_openai_sse_line(line: &str) -> Result> { - let line = line.trim(); - let Some(data) = line.strip_prefix("data:") else { - return Ok(None); - }; - let data = data.trim(); - if data == "[DONE]" { - return Ok(Some(ProviderStreamEvent { - content: String::new(), - done: true, - })); - } - let payload = serde_json::from_str::(data) - .context("failed to parse local provider SSE data")?; - let content = payload - .get("choices") - .and_then(serde_json::Value::as_array) - .and_then(|choices| choices.first()) - .and_then(|choice| choice.get("delta")) - .and_then(|delta| delta.get("content")) - .and_then(serde_json::Value::as_str) - .or_else(|| { - payload - .get("choices") - .and_then(serde_json::Value::as_array) - .and_then(|choices| choices.first()) - .and_then(|choice| choice.get("message")) - .and_then(|message| message.get("content")) - .and_then(serde_json::Value::as_str) - }); - Ok(content.map(|content| ProviderStreamEvent { - content: content.to_owned(), - done: false, - })) -} - -fn parse_anthropic_sse_line(line: &str) -> Result> { - let line = line.trim(); - let Some(data) = line.strip_prefix("data:") else { - return Ok(None); - }; - let data = data.trim(); - if data == "[DONE]" { - return Ok(Some(ProviderStreamEvent { - content: String::new(), - done: true, - })); - } - let payload = serde_json::from_str::(data) - .context("failed to parse Anthropic provider SSE data")?; - match payload.get("type").and_then(serde_json::Value::as_str) { - Some("content_block_delta") => Ok(payload - .get("delta") - .and_then(|delta| delta.get("text")) - .and_then(serde_json::Value::as_str) - .map(|content| ProviderStreamEvent { - content: content.to_owned(), - done: false, - })), - Some("content_block_start") => Ok(payload - .get("content_block") - .and_then(|block| block.get("text")) - .and_then(serde_json::Value::as_str) - .filter(|content| !content.is_empty()) - .map(|content| ProviderStreamEvent { - content: content.to_owned(), - done: false, - })), - Some("message_stop") => Ok(Some(ProviderStreamEvent { - content: String::new(), - done: true, - })), - Some("error") => { - let message = payload - .get("error") - .and_then(|error| error.get("message")) - .and_then(serde_json::Value::as_str) - .unwrap_or("unknown Anthropic stream error"); - bail!("Anthropic provider stream failed: {message}"); - } - _ => Ok(None), - } -} - #[cfg(test)] mod tests { use super::*; @@ -1396,97 +901,6 @@ mod tests { Ok(()) } - #[test] - fn parses_openai_chat_content() -> Result<()> { - let payload = serde_json::json!({ - "choices": [ - { - "message": { - "content": "hello from local" - } - } - ] - }); - assert_eq!(parse_openai_chat_content(&payload)?, "hello from local"); - Ok(()) - } - - #[test] - fn parses_openai_chat_tool_calls_without_text() -> Result<()> { - let payload = serde_json::json!({ - "choices": [ - { - "message": { - "tool_calls": [ - { - "id": "call-1", - "type": "function", - "function": { - "name": "install_sdk_dry_run", - "arguments": "{\"channel\":\"release\",\"format\":\"pip\"}" - } - } - ] - } - } - ] - }); - let (content, calls) = parse_openai_chat_message(&payload)?; - assert!(content.is_empty()); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].id.as_deref(), Some("call-1")); - assert_eq!(calls[0].name, "install_sdk_dry_run"); - assert_eq!(calls[0].arguments["channel"], "release"); - let error = parse_openai_chat_content(&payload).unwrap_err().to_string(); - assert!(error.contains("tool calls but no text content")); - Ok(()) - } - - #[test] - fn parses_anthropic_chat_content() -> Result<()> { - let payload = serde_json::json!({ - "content": [ - {"type": "text", "text": "hello "}, - {"type": "text", "text": "from anthropic"} - ] - }); - assert_eq!( - parse_anthropic_chat_content(&payload)?, - "hello from anthropic" - ); - Ok(()) - } - - #[test] - fn parses_anthropic_tool_use_without_text() -> Result<()> { - let payload = serde_json::json!({ - "content": [ - { - "type": "tool_use", - "id": "toolu_1", - "name": "install_sdk_dry_run", - "input": { - "channel": "release", - "format": "wheel" - } - } - ] - }); - - let (content, calls) = parse_anthropic_chat_message(&payload)?; - - assert!(content.is_empty()); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].id.as_deref(), Some("toolu_1")); - assert_eq!(calls[0].name, "install_sdk_dry_run"); - assert_eq!(calls[0].arguments["channel"], "release"); - let error = parse_anthropic_chat_content(&payload) - .unwrap_err() - .to_string(); - assert!(error.contains("tool calls but no text content")); - Ok(()) - } - #[test] fn remote_model_prefers_request_model() -> Result<()> { assert_eq!( @@ -1496,231 +910,6 @@ mod tests { Ok(()) } - #[test] - fn openai_and_anthropic_request_bodies_include_model_and_messages() { - let request = ChatRequest { - model: Some("model-a".to_owned()), - messages: vec![ChatMessage { - role: "user".to_owned(), - content: "hello".to_owned(), - }], - max_tokens: Some(42), - rocm_tools: false, - ..Default::default() - }; - let openai = openai_chat_request_body("model-a", &request); - assert_eq!(openai["model"], "model-a"); - assert_eq!(openai["messages"][0]["content"], "hello"); - assert_eq!(openai["stream"], false); - assert!(openai.get("tools").is_none()); - let anthropic = anthropic_chat_request_body("claude-test", &request); - assert_eq!(anthropic["model"], "claude-test"); - assert_eq!(anthropic["messages"][0]["role"], "user"); - assert_eq!(anthropic["max_tokens"], 42); - } - - #[test] - fn request_bodies_include_temperature_and_top_p_only_when_set() { - let base = ChatRequest { - model: Some("model-a".to_owned()), - messages: vec![ChatMessage { - role: "user".to_owned(), - content: "hello".to_owned(), - }], - max_tokens: Some(42), - rocm_tools: false, - ..Default::default() - }; - - // Omitted by default so provider/server defaults are preserved. - let openai_default = openai_chat_request_body("model-a", &base); - assert!(openai_default.get("temperature").is_none()); - assert!(openai_default.get("top_p").is_none()); - let anthropic_default = anthropic_chat_request_body("claude-test", &base); - assert!(anthropic_default.get("temperature").is_none()); - assert!(anthropic_default.get("top_p").is_none()); - - let tuned = ChatRequest { - temperature: Some(0.25), - top_p: Some(0.5), - ..base - }; - let openai = openai_chat_request_body("model-a", &tuned); - assert_eq!(openai["temperature"], 0.25); - assert_eq!(openai["top_p"], 0.5); - let anthropic = anthropic_chat_request_body("claude-test", &tuned); - assert_eq!(anthropic["temperature"], 0.25); - assert_eq!(anthropic["top_p"], 0.5); - } - - #[test] - fn openai_local_qwen3_request_disables_thinking_for_visible_answers() { - let request = ChatRequest { - model: Some("Qwen3-0.6B-GGUF".to_owned()), - messages: vec![ChatMessage { - role: "user".to_owned(), - content: "hello".to_owned(), - }], - max_tokens: Some(16), - rocm_tools: false, - ..Default::default() - }; - - let body = openai_local_chat_request_body("Qwen3-0.6B-GGUF", &request); - - assert_eq!(body["chat_template_kwargs"]["enable_thinking"], false); - } - - #[test] - fn openai_request_body_includes_rocm_tools_only_for_non_streaming_tool_mode() { - let request = ChatRequest { - model: Some("model-a".to_owned()), - messages: vec![ChatMessage { - role: "user".to_owned(), - content: "check this host".to_owned(), - }], - max_tokens: Some(42), - rocm_tools: true, - ..Default::default() - }; - let non_streaming = openai_chat_request_body("model-a", &request); - assert_eq!(non_streaming["tool_choice"], "auto"); - let tools = non_streaming["tools"] - .as_array() - .expect("tools should be set"); - let tool_names = tools - .iter() - .filter_map(|tool| { - tool.get("function") - .and_then(|function| function.get("name")) - .and_then(serde_json::Value::as_str) - }) - .collect::>(); - for name in [ - "bridge_snapshot", - "service_logs", - "natural_language_plan", - "path_exists", - "port_status", - "rocm_command", - "update_check", - "install_sdk_dry_run", - ] { - assert!( - tool_names.contains(&name), - "missing ROCm chat tool `{name}`" - ); - } - for name in [ - "install_sdk", - "install_engine", - "launch_server", - "stop_server", - "watcher_enable", - "watcher_disable", - ] { - assert!( - !tool_names.contains(&name), - "mutating ROCm chat tool `{name}` should be requested through rocm_command" - ); - } - - let streaming = openai_stream_chat_request_body("model-a", &request); - assert!(streaming.get("tools").is_none()); - assert!(streaming.get("tool_choice").is_none()); - } - - #[test] - fn anthropic_request_body_includes_rocm_tools_and_system_only_for_non_streaming_tool_mode() { - let request = ChatRequest { - model: Some("model-a".to_owned()), - messages: vec![ - ChatMessage { - role: "system".to_owned(), - content: "Use ROCm tools only when useful.".to_owned(), - }, - ChatMessage { - role: "user".to_owned(), - content: "check this host".to_owned(), - }, - ], - max_tokens: Some(42), - rocm_tools: true, - ..Default::default() - }; - - let non_streaming = anthropic_chat_request_body("claude-test", &request); - - assert_eq!(non_streaming["system"], "Use ROCm tools only when useful."); - assert_eq!(non_streaming["messages"][0]["role"], "user"); - assert_eq!(non_streaming["messages"][0]["content"], "check this host"); - assert_eq!(non_streaming["messages"].as_array().unwrap().len(), 1); - let tools = non_streaming["tools"] - .as_array() - .expect("tools should be set"); - let tool_names = tools - .iter() - .filter_map(|tool| tool.get("name").and_then(serde_json::Value::as_str)) - .collect::>(); - for name in [ - "bridge_snapshot", - "service_logs", - "natural_language_plan", - "path_exists", - "port_status", - "rocm_command", - "update_check", - "install_sdk_dry_run", - ] { - assert!( - tool_names.contains(&name), - "missing ROCm chat tool `{name}`" - ); - } - for name in [ - "install_sdk", - "install_engine", - "launch_server", - "stop_server", - "watcher_enable", - "watcher_disable", - ] { - assert!( - !tool_names.contains(&name), - "mutating ROCm chat tool `{name}` should be requested through rocm_command" - ); - } - assert!(tools.iter().any(|tool| { - tool.get("name").and_then(serde_json::Value::as_str) == Some("install_sdk_dry_run") - && tool - .get("input_schema") - .and_then(|schema| schema.get("type")) - .and_then(serde_json::Value::as_str) - == Some("object") - && tool - .get("input_schema") - .and_then(|schema| schema.get("properties")) - .and_then(|properties| properties.get("build_date")) - .is_some() - })); - - let streaming = anthropic_stream_chat_request_body("claude-test", &request); - assert!(streaming.get("tools").is_none()); - assert_eq!(streaming["system"], "Use ROCm tools only when useful."); - } - - #[test] - fn parses_openai_sse_chat_events() -> Result<()> { - let body = "data: {\"choices\":[{\"delta\":{\"content\":\"hel\"}}]}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\n\ndata: [DONE]\n\n"; - let events = parse_openai_sse_chat_events(body)?; - assert_eq!(events.len(), 3); - assert_eq!(events[0].content, "hel"); - assert!(!events[0].done); - assert_eq!(events[1].content, "lo"); - assert!(events[2].done); - Ok(()) - } - #[test] fn local_provider_chat_posts_to_ready_service() -> Result<()> { let (root, paths) = temp_app_paths("local-provider-chat"); @@ -3117,6 +2306,3 @@ mod tests { crate::test_support::assert_each_call_gets_its_own_root(|| temp_app_paths("same-label").0); } } - -#[cfg(test)] -mod sse_emitter_proptests; diff --git a/apps/rocm/src/providers/local_http.rs b/apps/rocm/src/providers/local_http.rs index ce87ea5e8e..947b6af529 100644 --- a/apps/rocm/src/providers/local_http.rs +++ b/apps/rocm/src/providers/local_http.rs @@ -12,7 +12,8 @@ //! reads `OpenAiSseEmitter`; that is the only edge back into `providers` //! besides its shared types. -use super::{OpenAiSseEmitter, ProviderStreamEvent, SseBytesEmitter}; +use super::wire::OpenAiSseEmitter; +use super::{ProviderStreamEvent, SseBytesEmitter}; use anyhow::{Context, Result, bail}; use rocm_core::{ connect_tcp_stream, format_host_port, read_http_response_bounded, write_all_tcp_stream, diff --git a/apps/rocm/src/providers/local_http/proptests.rs b/apps/rocm/src/providers/local_http/proptests.rs index af0951f30a..0300a8b7e6 100644 --- a/apps/rocm/src/providers/local_http/proptests.rs +++ b/apps/rocm/src/providers/local_http/proptests.rs @@ -16,7 +16,7 @@ use proptest::prelude::*; use std::io::{BufReader, Cursor, ErrorKind, Read}; const OPENAI_FIXTURES: &[&str] = &[ - // providers.rs `parses_openai_sse_chat_events`. + // providers/wire.rs `parses_openai_sse_chat_events`. "data: {\"choices\":[{\"delta\":{\"content\":\"hel\"}}]}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\n\ndata: [DONE]\n\n", // Same body, CRLF line endings. "data: {\"choices\":[{\"delta\":{\"content\":\"hel\"}}]}\r\n\r\ndata: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\r\n\r\ndata: [DONE]\r\n\r\n", diff --git a/apps/rocm/src/providers/wire.rs b/apps/rocm/src/providers/wire.rs new file mode 100644 index 0000000000..e30a6ae433 --- /dev/null +++ b/apps/rocm/src/providers/wire.rs @@ -0,0 +1,855 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The provider wire formats: the OpenAI-compatible and Anthropic chat request +//! bodies (including the local-model request defaults), the parsers for their +//! non-streaming responses, and the SSE emitters that turn a streamed response +//! into `ProviderStreamEvent`s. +//! +//! Depends on the shared request/response types in `providers` and on the ROCm +//! tool schema in `tool_schema`; the local HTTP transport in `local_http` +//! reads [`OpenAiSseEmitter`] from here, so the dependency is one-way. + +use super::tool_schema::{rocm_anthropic_tool_definitions, rocm_openai_tool_definitions}; +use super::{ChatMessage, ChatRequest, ChatToolCall, ProviderStreamEvent, SseBytesEmitter}; +use anyhow::{Context, Result, bail}; + +pub(super) fn openai_chat_request_body(model: &str, request: &ChatRequest) -> serde_json::Value { + openai_chat_request_body_with_stream(model, request, false) +} + +pub(super) fn openai_stream_chat_request_body( + model: &str, + request: &ChatRequest, +) -> serde_json::Value { + openai_chat_request_body_with_stream(model, request, true) +} + +pub(super) fn openai_local_chat_request_body( + model: &str, + request: &ChatRequest, +) -> serde_json::Value { + let request = local_openai_compatible_request(request); + let mut body = openai_chat_request_body(model, &request); + apply_local_model_request_defaults(model, &mut body); + body +} + +pub(super) fn openai_local_stream_chat_request_body( + model: &str, + request: &ChatRequest, +) -> serde_json::Value { + let request = local_openai_compatible_request(request); + let mut body = openai_stream_chat_request_body(model, &request); + apply_local_model_request_defaults(model, &mut body); + body +} + +fn apply_local_model_request_defaults(model: &str, body: &mut serde_json::Value) { + if local_model_should_disable_thinking(model) { + body["chat_template_kwargs"] = serde_json::json!({ + "enable_thinking": false + }); + } +} + +fn local_model_should_disable_thinking(model: &str) -> bool { + let normalized = model.to_ascii_lowercase(); + normalized.contains("qwen3") || normalized.contains("qwen/qwen3") +} + +fn local_openai_compatible_request(request: &ChatRequest) -> ChatRequest { + let system = request + .messages + .iter() + .filter(|message| message.role == "system") + .map(|message| message.content.trim()) + .filter(|content| !content.is_empty()) + .collect::>() + .join("\n\n"); + let mut messages = request + .messages + .iter() + .filter(|message| message.role != "system") + .cloned() + .collect::>(); + if !system.is_empty() { + if let Some(first_user) = messages.iter_mut().find(|message| message.role == "user") { + first_user.content = format!( + "ROCm CLI guidance:\n{system}\n\nUser message:\n{}", + first_user.content + ); + } else { + messages.insert( + 0, + ChatMessage { + role: "user".to_owned(), + content: format!("ROCm CLI guidance:\n{system}"), + }, + ); + } + } + ChatRequest { + model: request.model.clone(), + messages, + max_tokens: request.max_tokens, + temperature: request.temperature, + top_p: request.top_p, + rocm_tools: request.rocm_tools, + } +} + +fn openai_chat_request_body_with_stream( + model: &str, + request: &ChatRequest, + stream: bool, +) -> serde_json::Value { + let mut body = serde_json::json!({ + "model": model, + "messages": request.messages.iter().map(|message| { + serde_json::json!({ + "role": message.role, + "content": message.content, + }) + }).collect::>(), + "stream": stream, + "max_tokens": request.max_tokens.unwrap_or(512), + }); + if let Some(temperature) = request.temperature { + body["temperature"] = serde_json::json!(temperature); + } + if let Some(top_p) = request.top_p { + body["top_p"] = serde_json::json!(top_p); + } + if request.rocm_tools && !stream { + body["tools"] = serde_json::Value::Array(rocm_openai_tool_definitions()); + body["tool_choice"] = serde_json::json!("auto"); + } + body +} + +pub(super) fn anthropic_chat_request_body(model: &str, request: &ChatRequest) -> serde_json::Value { + anthropic_chat_request_body_with_stream(model, request, false) +} + +pub(super) fn anthropic_stream_chat_request_body( + model: &str, + request: &ChatRequest, +) -> serde_json::Value { + anthropic_chat_request_body_with_stream(model, request, true) +} + +fn anthropic_chat_request_body_with_stream( + model: &str, + request: &ChatRequest, + stream: bool, +) -> serde_json::Value { + let mut body = serde_json::json!({ + "model": model, + "messages": request.messages.iter().filter(|message| message.role != "system").map(|message| { + serde_json::json!({ + "role": if message.role == "assistant" { "assistant" } else { "user" }, + "content": message.content, + }) + }).collect::>(), + "max_tokens": request.max_tokens.unwrap_or(512), + "stream": stream, + }); + let system = request + .messages + .iter() + .filter(|message| message.role == "system") + .map(|message| message.content.trim()) + .filter(|content| !content.is_empty()) + .collect::>() + .join("\n\n"); + if !system.is_empty() { + body["system"] = serde_json::Value::String(system); + } + if let Some(temperature) = request.temperature { + body["temperature"] = serde_json::json!(temperature); + } + if let Some(top_p) = request.top_p { + body["top_p"] = serde_json::json!(top_p); + } + if request.rocm_tools && !stream { + body["tools"] = serde_json::Value::Array(rocm_anthropic_tool_definitions()); + } + body +} + +#[cfg(test)] +fn parse_openai_chat_content(response: &serde_json::Value) -> Result { + let (content, tool_calls) = parse_openai_chat_message(response)?; + if content.is_empty() && !tool_calls.is_empty() { + bail!("OpenAI-compatible response contained tool calls but no text content"); + } + Ok(content) +} + +pub(super) fn parse_openai_chat_message( + response: &serde_json::Value, +) -> Result<(String, Vec)> { + let message = response + .get("choices") + .and_then(serde_json::Value::as_array) + .and_then(|choices| choices.first()) + .and_then(|choice| choice.get("message")) + .context("OpenAI-compatible response did not include choices[0].message")?; + let content = message + .get("content") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .unwrap_or_default(); + let tool_calls = parse_openai_tool_calls(message)?; + if content.is_empty() && tool_calls.is_empty() { + bail!("OpenAI-compatible response did not include message content or tool calls"); + } + Ok((content, tool_calls)) +} + +fn parse_openai_tool_calls(message: &serde_json::Value) -> Result> { + let Some(items) = message + .get("tool_calls") + .and_then(serde_json::Value::as_array) + else { + return Ok(Vec::new()); + }; + let mut calls = Vec::new(); + for item in items { + let function = item + .get("function") + .context("OpenAI-compatible tool call was missing function")?; + let name = function + .get("name") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .context("OpenAI-compatible tool call was missing function.name")?; + let arguments = match function.get("arguments") { + Some(serde_json::Value::String(text)) if text.trim().is_empty() => { + serde_json::json!({}) + } + Some(serde_json::Value::String(text)) => serde_json::from_str(text) + .with_context(|| format!("tool call `{name}` arguments were not valid JSON"))?, + Some(value) => value.clone(), + None => serde_json::json!({}), + }; + calls.push(ChatToolCall { + id: item + .get("id") + .and_then(serde_json::Value::as_str) + .map(str::to_owned), + name, + arguments, + }); + } + Ok(calls) +} + +#[cfg(test)] +fn parse_anthropic_chat_content(response: &serde_json::Value) -> Result { + let (content, tool_calls) = parse_anthropic_chat_message(response)?; + if content.is_empty() && !tool_calls.is_empty() { + bail!("anthropic provider response contained tool calls but no text content"); + } + Ok(content) +} + +pub(super) fn parse_anthropic_chat_message( + response: &serde_json::Value, +) -> Result<(String, Vec)> { + let items = response + .get("content") + .and_then(serde_json::Value::as_array) + .context("anthropic provider response did not include content blocks")?; + let mut text = String::new(); + let mut tool_calls = Vec::new(); + for item in items { + match item.get("type").and_then(serde_json::Value::as_str) { + Some("text") => { + if let Some(chunk) = item.get("text").and_then(serde_json::Value::as_str) { + text.push_str(chunk); + } + } + Some("tool_use") => { + let name = item + .get("name") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .context("anthropic provider tool_use block was missing name")?; + let arguments = item + .get("input") + .cloned() + .unwrap_or_else(|| serde_json::json!({})); + tool_calls.push(ChatToolCall { + id: item + .get("id") + .and_then(serde_json::Value::as_str) + .map(str::to_owned), + name, + arguments, + }); + } + _ => {} + } + } + if text.is_empty() && tool_calls.is_empty() { + bail!("anthropic provider response did not include text content or tool calls"); + } + Ok((text, tool_calls)) +} + +pub(super) struct OpenAiSseEmitter<'a> { + pending: Vec, + done: bool, + on_event: &'a mut dyn FnMut(ProviderStreamEvent) -> Result<()>, +} + +impl<'a> OpenAiSseEmitter<'a> { + pub(super) fn new(on_event: &'a mut dyn FnMut(ProviderStreamEvent) -> Result<()>) -> Self { + Self { + pending: Vec::new(), + done: false, + on_event, + } + } + + pub(super) fn finish(mut self) -> Result<()> { + if !self.pending.is_empty() { + self.emit_pending_line()?; + } + if !self.done { + (self.on_event)(ProviderStreamEvent { + content: String::new(), + done: true, + })?; + self.done = true; + } + Ok(()) + } + + fn emit_pending_line(&mut self) -> Result<()> { + let line = String::from_utf8_lossy(&self.pending) + .trim_end_matches('\r') + .to_owned(); + self.pending.clear(); + self.emit_line(&line) + } + + fn emit_line(&mut self, line: &str) -> Result<()> { + if self.done { + return Ok(()); + } + let Some(event) = parse_openai_sse_line(line)? else { + return Ok(()); + }; + if event.done { + self.done = true; + } + (self.on_event)(event) + } +} + +impl SseBytesEmitter for OpenAiSseEmitter<'_> { + fn push_bytes(&mut self, bytes: &[u8]) -> Result<()> { + for byte in bytes { + if *byte == b'\n' { + self.emit_pending_line()?; + } else { + self.pending.push(*byte); + } + } + Ok(()) + } +} + +pub(super) struct AnthropicSseEmitter<'a> { + pending: Vec, + done: bool, + on_event: &'a mut dyn FnMut(ProviderStreamEvent) -> Result<()>, +} + +impl<'a> AnthropicSseEmitter<'a> { + pub(super) fn new(on_event: &'a mut dyn FnMut(ProviderStreamEvent) -> Result<()>) -> Self { + Self { + pending: Vec::new(), + done: false, + on_event, + } + } + + pub(super) fn finish(mut self) -> Result<()> { + if !self.pending.is_empty() { + self.emit_pending_line()?; + } + if !self.done { + (self.on_event)(ProviderStreamEvent { + content: String::new(), + done: true, + })?; + self.done = true; + } + Ok(()) + } + + fn emit_pending_line(&mut self) -> Result<()> { + let line = String::from_utf8_lossy(&self.pending) + .trim_end_matches('\r') + .to_owned(); + self.pending.clear(); + self.emit_line(&line) + } + + fn emit_line(&mut self, line: &str) -> Result<()> { + if self.done { + return Ok(()); + } + let Some(event) = parse_anthropic_sse_line(line)? else { + return Ok(()); + }; + if event.done { + self.done = true; + } + (self.on_event)(event) + } +} + +impl SseBytesEmitter for AnthropicSseEmitter<'_> { + fn push_bytes(&mut self, bytes: &[u8]) -> Result<()> { + for byte in bytes { + if *byte == b'\n' { + self.emit_pending_line()?; + } else { + self.pending.push(*byte); + } + } + Ok(()) + } +} + +#[cfg(test)] +fn parse_openai_sse_chat_events(body: &str) -> Result> { + let mut events = Vec::new(); + for line in body.lines() { + let Some(event) = parse_openai_sse_line(line)? else { + continue; + }; + events.push(event); + } + if !events.iter().any(|event| event.done) { + events.push(ProviderStreamEvent { + content: String::new(), + done: true, + }); + } + Ok(events) +} + +fn parse_openai_sse_line(line: &str) -> Result> { + let line = line.trim(); + let Some(data) = line.strip_prefix("data:") else { + return Ok(None); + }; + let data = data.trim(); + if data == "[DONE]" { + return Ok(Some(ProviderStreamEvent { + content: String::new(), + done: true, + })); + } + let payload = serde_json::from_str::(data) + .context("failed to parse local provider SSE data")?; + let content = payload + .get("choices") + .and_then(serde_json::Value::as_array) + .and_then(|choices| choices.first()) + .and_then(|choice| choice.get("delta")) + .and_then(|delta| delta.get("content")) + .and_then(serde_json::Value::as_str) + .or_else(|| { + payload + .get("choices") + .and_then(serde_json::Value::as_array) + .and_then(|choices| choices.first()) + .and_then(|choice| choice.get("message")) + .and_then(|message| message.get("content")) + .and_then(serde_json::Value::as_str) + }); + Ok(content.map(|content| ProviderStreamEvent { + content: content.to_owned(), + done: false, + })) +} + +fn parse_anthropic_sse_line(line: &str) -> Result> { + let line = line.trim(); + let Some(data) = line.strip_prefix("data:") else { + return Ok(None); + }; + let data = data.trim(); + if data == "[DONE]" { + return Ok(Some(ProviderStreamEvent { + content: String::new(), + done: true, + })); + } + let payload = serde_json::from_str::(data) + .context("failed to parse Anthropic provider SSE data")?; + match payload.get("type").and_then(serde_json::Value::as_str) { + Some("content_block_delta") => Ok(payload + .get("delta") + .and_then(|delta| delta.get("text")) + .and_then(serde_json::Value::as_str) + .map(|content| ProviderStreamEvent { + content: content.to_owned(), + done: false, + })), + Some("content_block_start") => Ok(payload + .get("content_block") + .and_then(|block| block.get("text")) + .and_then(serde_json::Value::as_str) + .filter(|content| !content.is_empty()) + .map(|content| ProviderStreamEvent { + content: content.to_owned(), + done: false, + })), + Some("message_stop") => Ok(Some(ProviderStreamEvent { + content: String::new(), + done: true, + })), + Some("error") => { + let message = payload + .get("error") + .and_then(|error| error.get("message")) + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown Anthropic stream error"); + bail!("Anthropic provider stream failed: {message}"); + } + _ => Ok(None), + } +} + +#[cfg(test)] +mod sse_emitter_proptests; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_openai_chat_content() -> Result<()> { + let payload = serde_json::json!({ + "choices": [ + { + "message": { + "content": "hello from local" + } + } + ] + }); + assert_eq!(parse_openai_chat_content(&payload)?, "hello from local"); + Ok(()) + } + + #[test] + fn parses_openai_chat_tool_calls_without_text() -> Result<()> { + let payload = serde_json::json!({ + "choices": [ + { + "message": { + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": { + "name": "install_sdk_dry_run", + "arguments": "{\"channel\":\"release\",\"format\":\"pip\"}" + } + } + ] + } + } + ] + }); + let (content, calls) = parse_openai_chat_message(&payload)?; + assert!(content.is_empty()); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].id.as_deref(), Some("call-1")); + assert_eq!(calls[0].name, "install_sdk_dry_run"); + assert_eq!(calls[0].arguments["channel"], "release"); + let error = parse_openai_chat_content(&payload).unwrap_err().to_string(); + assert!(error.contains("tool calls but no text content")); + Ok(()) + } + + #[test] + fn parses_anthropic_chat_content() -> Result<()> { + let payload = serde_json::json!({ + "content": [ + {"type": "text", "text": "hello "}, + {"type": "text", "text": "from anthropic"} + ] + }); + assert_eq!( + parse_anthropic_chat_content(&payload)?, + "hello from anthropic" + ); + Ok(()) + } + + #[test] + fn parses_anthropic_tool_use_without_text() -> Result<()> { + let payload = serde_json::json!({ + "content": [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "install_sdk_dry_run", + "input": { + "channel": "release", + "format": "wheel" + } + } + ] + }); + + let (content, calls) = parse_anthropic_chat_message(&payload)?; + + assert!(content.is_empty()); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].id.as_deref(), Some("toolu_1")); + assert_eq!(calls[0].name, "install_sdk_dry_run"); + assert_eq!(calls[0].arguments["channel"], "release"); + let error = parse_anthropic_chat_content(&payload) + .unwrap_err() + .to_string(); + assert!(error.contains("tool calls but no text content")); + Ok(()) + } + + #[test] + fn openai_and_anthropic_request_bodies_include_model_and_messages() { + let request = ChatRequest { + model: Some("model-a".to_owned()), + messages: vec![ChatMessage { + role: "user".to_owned(), + content: "hello".to_owned(), + }], + max_tokens: Some(42), + rocm_tools: false, + ..Default::default() + }; + let openai = openai_chat_request_body("model-a", &request); + assert_eq!(openai["model"], "model-a"); + assert_eq!(openai["messages"][0]["content"], "hello"); + assert_eq!(openai["stream"], false); + assert!(openai.get("tools").is_none()); + let anthropic = anthropic_chat_request_body("claude-test", &request); + assert_eq!(anthropic["model"], "claude-test"); + assert_eq!(anthropic["messages"][0]["role"], "user"); + assert_eq!(anthropic["max_tokens"], 42); + } + + #[test] + fn request_bodies_include_temperature_and_top_p_only_when_set() { + let base = ChatRequest { + model: Some("model-a".to_owned()), + messages: vec![ChatMessage { + role: "user".to_owned(), + content: "hello".to_owned(), + }], + max_tokens: Some(42), + rocm_tools: false, + ..Default::default() + }; + + // Omitted by default so provider/server defaults are preserved. + let openai_default = openai_chat_request_body("model-a", &base); + assert!(openai_default.get("temperature").is_none()); + assert!(openai_default.get("top_p").is_none()); + let anthropic_default = anthropic_chat_request_body("claude-test", &base); + assert!(anthropic_default.get("temperature").is_none()); + assert!(anthropic_default.get("top_p").is_none()); + + let tuned = ChatRequest { + temperature: Some(0.25), + top_p: Some(0.5), + ..base + }; + let openai = openai_chat_request_body("model-a", &tuned); + assert_eq!(openai["temperature"], 0.25); + assert_eq!(openai["top_p"], 0.5); + let anthropic = anthropic_chat_request_body("claude-test", &tuned); + assert_eq!(anthropic["temperature"], 0.25); + assert_eq!(anthropic["top_p"], 0.5); + } + + #[test] + fn openai_local_qwen3_request_disables_thinking_for_visible_answers() { + let request = ChatRequest { + model: Some("Qwen3-0.6B-GGUF".to_owned()), + messages: vec![ChatMessage { + role: "user".to_owned(), + content: "hello".to_owned(), + }], + max_tokens: Some(16), + rocm_tools: false, + ..Default::default() + }; + + let body = openai_local_chat_request_body("Qwen3-0.6B-GGUF", &request); + + assert_eq!(body["chat_template_kwargs"]["enable_thinking"], false); + } + + #[test] + fn openai_request_body_includes_rocm_tools_only_for_non_streaming_tool_mode() { + let request = ChatRequest { + model: Some("model-a".to_owned()), + messages: vec![ChatMessage { + role: "user".to_owned(), + content: "check this host".to_owned(), + }], + max_tokens: Some(42), + rocm_tools: true, + ..Default::default() + }; + let non_streaming = openai_chat_request_body("model-a", &request); + assert_eq!(non_streaming["tool_choice"], "auto"); + let tools = non_streaming["tools"] + .as_array() + .expect("tools should be set"); + let tool_names = tools + .iter() + .filter_map(|tool| { + tool.get("function") + .and_then(|function| function.get("name")) + .and_then(serde_json::Value::as_str) + }) + .collect::>(); + for name in [ + "bridge_snapshot", + "service_logs", + "natural_language_plan", + "path_exists", + "port_status", + "rocm_command", + "update_check", + "install_sdk_dry_run", + ] { + assert!( + tool_names.contains(&name), + "missing ROCm chat tool `{name}`" + ); + } + for name in [ + "install_sdk", + "install_engine", + "launch_server", + "stop_server", + "watcher_enable", + "watcher_disable", + ] { + assert!( + !tool_names.contains(&name), + "mutating ROCm chat tool `{name}` should be requested through rocm_command" + ); + } + + let streaming = openai_stream_chat_request_body("model-a", &request); + assert!(streaming.get("tools").is_none()); + assert!(streaming.get("tool_choice").is_none()); + } + + #[test] + fn anthropic_request_body_includes_rocm_tools_and_system_only_for_non_streaming_tool_mode() { + let request = ChatRequest { + model: Some("model-a".to_owned()), + messages: vec![ + ChatMessage { + role: "system".to_owned(), + content: "Use ROCm tools only when useful.".to_owned(), + }, + ChatMessage { + role: "user".to_owned(), + content: "check this host".to_owned(), + }, + ], + max_tokens: Some(42), + rocm_tools: true, + ..Default::default() + }; + + let non_streaming = anthropic_chat_request_body("claude-test", &request); + + assert_eq!(non_streaming["system"], "Use ROCm tools only when useful."); + assert_eq!(non_streaming["messages"][0]["role"], "user"); + assert_eq!(non_streaming["messages"][0]["content"], "check this host"); + assert_eq!(non_streaming["messages"].as_array().unwrap().len(), 1); + let tools = non_streaming["tools"] + .as_array() + .expect("tools should be set"); + let tool_names = tools + .iter() + .filter_map(|tool| tool.get("name").and_then(serde_json::Value::as_str)) + .collect::>(); + for name in [ + "bridge_snapshot", + "service_logs", + "natural_language_plan", + "path_exists", + "port_status", + "rocm_command", + "update_check", + "install_sdk_dry_run", + ] { + assert!( + tool_names.contains(&name), + "missing ROCm chat tool `{name}`" + ); + } + for name in [ + "install_sdk", + "install_engine", + "launch_server", + "stop_server", + "watcher_enable", + "watcher_disable", + ] { + assert!( + !tool_names.contains(&name), + "mutating ROCm chat tool `{name}` should be requested through rocm_command" + ); + } + assert!(tools.iter().any(|tool| { + tool.get("name").and_then(serde_json::Value::as_str) == Some("install_sdk_dry_run") + && tool + .get("input_schema") + .and_then(|schema| schema.get("type")) + .and_then(serde_json::Value::as_str) + == Some("object") + && tool + .get("input_schema") + .and_then(|schema| schema.get("properties")) + .and_then(|properties| properties.get("build_date")) + .is_some() + })); + + let streaming = anthropic_stream_chat_request_body("claude-test", &request); + assert!(streaming.get("tools").is_none()); + assert_eq!(streaming["system"], "Use ROCm tools only when useful."); + } + + #[test] + fn parses_openai_sse_chat_events() -> Result<()> { + let body = "data: {\"choices\":[{\"delta\":{\"content\":\"hel\"}}]}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\n\ndata: [DONE]\n\n"; + let events = parse_openai_sse_chat_events(body)?; + assert_eq!(events.len(), 3); + assert_eq!(events[0].content, "hel"); + assert!(!events[0].done); + assert_eq!(events[1].content, "lo"); + assert!(events[2].done); + Ok(()) + } +} diff --git a/apps/rocm/src/providers/sse_emitter_proptests.rs b/apps/rocm/src/providers/wire/sse_emitter_proptests.rs similarity index 99% rename from apps/rocm/src/providers/sse_emitter_proptests.rs rename to apps/rocm/src/providers/wire/sse_emitter_proptests.rs index 46e9ec230d..77cabbb61a 100644 --- a/apps/rocm/src/providers/sse_emitter_proptests.rs +++ b/apps/rocm/src/providers/wire/sse_emitter_proptests.rs @@ -9,7 +9,7 @@ //! split of each body produces the events (and the error, for a body that //! fails) of the whole body, and the whole body produces the exact events //! each fixture lists. OpenAI fixture 1 is the body of -//! `parses_openai_sse_chat_events` in `providers.rs`. The others are written +//! `parses_openai_sse_chat_events` in `wire.rs`. The others are written //! here, in the shapes those servers send, to reach what that one body does //! not: CRLF line endings (what Python SSE servers emit by default), //! multi-byte UTF-8 content so that a split can land inside a UTF-8 sequence, @@ -30,7 +30,7 @@ //! field-reading branch, so the guard does not go quiet on one. //! - An ignored reproducer for #516 (in-band OpenAI error payloads). //! -//! The transport readers the sibling `proptests` module covers (chunked, +//! The transport readers that `local_http`'s `proptests` module covers (chunked, //! `Content-Length` and close-delimited bodies) feed only the OpenAI emitter, //! on the local-provider path. On the remote-provider path, the only one the //! Anthropic emitter has, both emitters are fed by `stream_json_with_headers` @@ -57,7 +57,7 @@ type Fixture = ( ); const OPENAI_FIXTURES: &[Fixture] = &[ - // providers.rs `parses_openai_sse_chat_events`. + // providers/wire.rs `parses_openai_sse_chat_events`. ( "data: {\"choices\":[{\"delta\":{\"content\":\"hel\"}}]}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\n\ndata: [DONE]\n\n", &[("hel", false), ("lo", false), ("", true)], diff --git a/docs/architecture.md b/docs/architecture.md index 3bbb0a9c06..9771666987 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -61,9 +61,10 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat.rs`](../apps/rocm/src/chat.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat.rs`](../apps/rocm/src/chat.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. - [`model_fit.rs`](../apps/rocm/src/model_fit.rs) — mechanical relocation, owns a `pub(crate)` type: answering whether a host can run a model for `rocm diagnose --model` (`assess_model_on_this_host`, `assess_model_with_host_paths`, `assess_model_for_host`), the accelerator-memory fact it rests on (`host_accelerator_memory`, `classify_accelerator_memory`, `examination_reports_amd_gpu`) and the platform gate that rules an engine out (`PlatformRuleOut`, `engine_ruled_out_by_platform`, `unsupported_here_for`). It is a module of its own, rather than part of the `diagnose` handler, so [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs), which reads `PlatformRuleOut` and `engine_ruled_out_by_platform`, does not depend on a command module. It depends on [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`select_serve_engine`) and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs), neither of which depends on it. `assess_model_on_this_host` (called from [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` are `pub(crate)`, and [`main.rs`](../apps/rocm/src/main.rs) re-imports the last two so `examine_report_text.rs` reaches them as `crate::`; every other item is private. The cluster's regression tests live in this module's own `mod tests`; the one that re-runs the test binary on a helper test by its full path (`engine_ruled_out_by_platform_gates_aim_on_docker_reachability`) names the helper's new path, `model_fit::tests::docker_unreachable_subprocess_helper`. - [`provider_keys.rs`](../apps/rocm/src/provider_keys.rs) — full domain extraction. -- [`providers.rs`](../apps/rocm/src/providers.rs) — full domain extraction (no dedicated command enum; invoked from the existing chat/config command flows). Keeps the request/response types, the `ProviderAdapter` trait and the `provider_*` dispatch, the `LocalProvider` and `RemoteProvider` implementations, local-service selection (`select_local_chat_service`, `ready_local_services`), the remote model and endpoint resolution, the `ureq` remote transport and the `SseBytesEmitter` trait. The ROCm tool schema and the local HTTP/SSE transport live in child modules under `providers/`, below. -- [`providers/tool_schema.rs`](../apps/rocm/src/providers/tool_schema.rs) — full domain extraction, a leaf child of `providers.rs`: the ROCm tool catalog a provider advertises to a model, as OpenAI function definitions (`rocm_openai_tool_definitions`) and reshaped for Anthropic's `tools` field (`rocm_anthropic_tool_definitions`). Both are `pub(super)`, read by the request-body builders in `providers.rs`; `rocm_openai_tool`, the one-tool constructor behind them, is private. Depends on nothing else in `providers`. -- [`providers/local_http.rs`](../apps/rocm/src/providers/local_http.rs) — full domain extraction, a child of `providers.rs`: the local provider's hand-rolled HTTP/1.1 transport. `post_json_to_local_endpoint` (with `post_json_to_local_endpoint_body` and `local_bearer_header`) is the blocking JSON POST; `stream_json_from_local_endpoint` is the streamed POST, which reads the response in whichever body framing the engine answers with (`read_sse_body` picks between `read_chunked_sse_body`, `read_fixed_sse_body` and `read_close_delimited_sse_body` from the headers, via `header_value`) and feeds an `OpenAiSseEmitter`; `body_truncated_at_eof` and `LOCAL_STREAM_TRUNCATED` name the failure when an engine closes the connection mid-answer; `parse_http_endpoint` reads the `http://` endpoint URL. `post_json_to_local_endpoint` and `stream_json_from_local_endpoint` are `pub(super)`, called from `LocalProvider`, and `LOCAL_STREAM_TRUNCATED` is also `pub(super)` (imported by `providers.rs` for its test module only); the rest is private. Its edge back into `providers` is the emitter and the `SseBytesEmitter` trait it pushes bytes through, plus the shared `ProviderStreamEvent` type. [`providers/local_http/proptests.rs`](../apps/rocm/src/providers/local_http/proptests.rs) holds the property tests of the SSE body readers and is declared from this module so they sit with the readers they test; the example-based test of `parse_http_endpoint` is in this module's own `mod tests`, and the tests that drive `LocalProvider` end to end against a mock server stay in `providers.rs`'s `mod tests` with their server helpers. +- [`providers.rs`](../apps/rocm/src/providers.rs) — full domain extraction (no dedicated command enum; invoked from the existing chat/config command flows). Keeps the request/response types, the `ProviderAdapter` trait and the `provider_*` dispatch, the `LocalProvider` and `RemoteProvider` implementations, local-service selection (`select_local_chat_service`, `ready_local_services`), the remote model and endpoint resolution, the `ureq` remote transport and the `SseBytesEmitter` trait. The ROCm tool schema, the local HTTP/SSE transport and the provider wire formats live in child modules under `providers/`, below; `providers.rs` imports the request-body builders, response parsers and SSE emitters it calls from `providers/wire.rs`. +- [`providers/tool_schema.rs`](../apps/rocm/src/providers/tool_schema.rs) — full domain extraction, a leaf child of `providers.rs`: the ROCm tool catalog a provider advertises to a model, as OpenAI function definitions (`rocm_openai_tool_definitions`) and reshaped for Anthropic's `tools` field (`rocm_anthropic_tool_definitions`). Both are `pub(super)`, read by the request-body builders in `providers/wire.rs`; `rocm_openai_tool`, the one-tool constructor behind them, is private. Depends on nothing else in `providers`. +- [`providers/local_http.rs`](../apps/rocm/src/providers/local_http.rs) — full domain extraction, a child of `providers.rs`: the local provider's hand-rolled HTTP/1.1 transport. `post_json_to_local_endpoint` (with `post_json_to_local_endpoint_body` and `local_bearer_header`) is the blocking JSON POST; `stream_json_from_local_endpoint` is the streamed POST, which reads the response in whichever body framing the engine answers with (`read_sse_body` picks between `read_chunked_sse_body`, `read_fixed_sse_body` and `read_close_delimited_sse_body` from the headers, via `header_value`) and feeds an `OpenAiSseEmitter`; `body_truncated_at_eof` and `LOCAL_STREAM_TRUNCATED` name the failure when an engine closes the connection mid-answer; `parse_http_endpoint` reads the `http://` endpoint URL. `post_json_to_local_endpoint` and `stream_json_from_local_endpoint` are `pub(super)`, called from `LocalProvider`, and `LOCAL_STREAM_TRUNCATED` is also `pub(super)` (imported by `providers.rs` for its test module only); the rest is private. Its edges back into `providers` are `OpenAiSseEmitter` from `providers/wire.rs`, the `SseBytesEmitter` trait the emitter is pushed bytes through (which stays in `providers.rs`), and the shared `ProviderStreamEvent` type; nothing in `wire.rs` reads `local_http.rs`, so the edge is one-way. [`providers/local_http/proptests.rs`](../apps/rocm/src/providers/local_http/proptests.rs) holds the property tests of the SSE body readers and is declared from this module so they sit with the readers they test; the example-based test of `parse_http_endpoint` is in this module's own `mod tests`, and the tests that drive `LocalProvider` end to end against a mock server stay in `providers.rs`'s `mod tests` with their server helpers. +- [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs) — full domain extraction, a child of `providers.rs`: the OpenAI-compatible and Anthropic wire formats. The request-body builders (`openai_chat_request_body`, `openai_stream_chat_request_body`, `openai_local_chat_request_body`, `openai_local_stream_chat_request_body`, `anthropic_chat_request_body`, `anthropic_stream_chat_request_body`, and the helpers behind them, including `local_openai_compatible_request`, `apply_local_model_request_defaults` and `local_model_should_disable_thinking`), the parsers for non-streaming responses (`parse_openai_chat_message`, `parse_anthropic_chat_message`), and the SSE emitters with their line parsers (`OpenAiSseEmitter` and `AnthropicSseEmitter`, which implement the `SseBytesEmitter` trait that stays in `providers.rs`; `parse_openai_sse_line`, `parse_anthropic_sse_line`). The builders, the two chat-message parsers and the two emitters (with their `new` and `finish`) are `pub(super)`, called from `LocalProvider` and `RemoteProvider`; `OpenAiSseEmitter` is also read by `providers/local_http.rs`. Depends on the shared request/response types in `providers.rs` and on `providers/tool_schema.rs`, and on nothing else. The tests of the builders and parsers live in this module's own `mod tests`, with the tool-catalog assertions that read through the request bodies, and [`providers/wire/sse_emitter_proptests.rs`](../apps/rocm/src/providers/wire/sse_emitter_proptests.rs), the split-invariance and panic-guard properties of the emitters and line parsers, is declared from this module; the stream tests that run an emitter against a mock server stay in `providers.rs`'s `mod tests`. - [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. - [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install: the orchestration (`finish_sdk_install`, `settle_engine_install`, `finalize_successful_sdk_install`), the engine-dependency check, the runtime device check and the post-install verdict built on it, and `SdkInstallConsents`. The torch alignment and retention it runs live in the child module [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs). `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`; `settle_engine_install` is `pub(crate)` and read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs). The host system-package installs live in [`system_packages.rs`](../apps/rocm/src/system_packages.rs); this module re-exports `ensure_libatomic_for_torch`/`ensure_libnuma_for_torch` (called from both `install_cmd.rs`'s `install()` and `engines_cmd.rs`) and `ensure_openmpi_for_vllm` (called from `engines_cmd.rs` and, for the engine auto-install after an SDK install, from here) so those callers keep their imports. - [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) — full domain extraction, a child of `runtime_install_finalize.rs` that depends on its parent's device check (`RuntimeDeviceCheck`, `device_check_verdict`) and on `runtime_manifest_for_selector` in [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): what to do about the torch an engine install finds in a managed runtime. `plan_torch_alignment` decides whether the SDK's torch must be realigned to the engine's pin, `align_runtime_torch` runs the realignment, `classify_retained_torch` decides when a torch that already works is kept as an intended divergence instead, and the `render_*` and `report_*` functions with `TorchAlignment` and `TorchRetention` print and log the outcome. The manifest readers (`sdk_torch_build_for_key`, `sdk_torch_build_from_manifest`, `runtime_python_for_key`, `runtime_library_paths_for_key`, `runtime_index_url_for_key`) live here too. The parent stays the caller (`settle_engine_install` and `maybe_auto_install_sdk_preferred_engine`), so it could not be a crate-level sibling without a cycle with the parent's device check. The items the parent calls are `pub(super)`: `TorchAlignment`, `TorchRetention`, `report_torch_alignment`, `report_torch_retention`, `classify_retained_torch`, `retained_diverged_package`, `deliberately_diverged_package`, `sdk_torch_build_for_key`, `runtime_python_for_key` and `runtime_library_paths_for_key`; the rest is private. Its tests cover planning, rendering, the opt-out and the retention rules, and reach the parent's private engine-dependency check as a descendant of it. From e023bd128cd59fc55f4549c4462c603fb453135d Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 15:59:53 +0000 Subject: [PATCH 4/4] test(providers): keep the saved proptest seeds replaying after the move proptest looks up its saved-failure file by the test's source path, so after providers/proptests.rs moved to providers/local_http/proptests.rs the three seeds in proptest-regressions/providers/proptests.txt were no longer replayed, silently. Move the file to the path that matches the new source location. The file's contents are unchanged. Checked that the new path is the one read: with a deliberately unparsable line appended, the local_http property tests printed 'unparsable line, ignoring' for the new path; the line was removed again. Signed-off-by: Jussi Elo --- .../proptest-regressions/providers/{ => local_http}/proptests.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename apps/rocm/proptest-regressions/providers/{ => local_http}/proptests.txt (100%) diff --git a/apps/rocm/proptest-regressions/providers/proptests.txt b/apps/rocm/proptest-regressions/providers/local_http/proptests.txt similarity index 100% rename from apps/rocm/proptest-regressions/providers/proptests.txt rename to apps/rocm/proptest-regressions/providers/local_http/proptests.txt