From 9ac33b1a39589178aa8eacdd34dab3fe496eff80 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 22 Oct 2026 10:21:38 +0000 Subject: [PATCH 1/7] feat: serve API docs for every running model server Of the servers rocm-cli runs, only vLLM generates API docs; Lemonade and the llama.cpp server behind it serve none, so a user of `rocm serve` gets an endpoint URL and no reference for what it accepts. The background helper `rocm serve` already starts now serves docs for every running model server on 127.0.0.1 (port 11430, `api_docs` in config.json): an OpenAPI description per server built with utoipa from the routes each engine reports, rendered by Swagger UI (vendored, works offline). "Try it out" goes through a same-origin forwarder to the server, so it works on every engine whatever its cross-origin settings. The forwarder bypasses proxies, passes on only Content-Type, Accept and Authorization, and refuses non-loopback Host and cross-site Origin headers. `rocm serve` and `rocm services list` print a `docs` link, but only once the docs answer with that server's description, so a link is never printed for docs that are off, a taken port, or an older helper. Signed-off-by: Roman Inflianskas --- Cargo.lock | 147 ++- MANIFEST.md | 19 + README.md | 39 +- THIRD_PARTY_NOTICES.txt | 341 ++++++ about.hbs | 127 ++ apps/rocm/src/api_docs_link.rs | 281 +++++ apps/rocm/src/main.rs | 21 +- apps/rocm/src/providers.rs | 1 + apps/rocm/src/serve_cmd.rs | 11 +- apps/rocm/src/serve_summary.rs | 42 +- apps/rocm/src/service_launch.rs | 47 +- apps/rocmd/Cargo.toml | 9 + apps/rocmd/src/api_docs.rs | 1078 +++++++++++++++++ apps/rocmd/src/lib.rs | 1 + apps/rocmd/src/service.rs | 43 +- crates/rocm-core/src/config.rs | 62 + crates/rocm-core/src/lib.rs | 9 +- crates/rocm-engine-protocol/src/lib.rs | 16 + docs/architecture.md | 2 + docs/manual-testing.md | 21 + docs/testing.md | 34 + engines/lemonade/src/lib.rs | 7 +- engines/vllm/src/state.rs | 7 +- .../features/model_serving.feature | 2 + .../features/service_docs.feature | 51 + tests/e2e-cucumber/tests/e2e.rs | 16 + .../tests/e2e/service_docs_steps.rs | 312 +++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 4 + tests/e2e-cucumber/tests/feature_naming.rs | 1 + 29 files changed, 2709 insertions(+), 42 deletions(-) create mode 100644 apps/rocm/src/api_docs_link.rs create mode 100644 apps/rocmd/src/api_docs.rs create mode 100644 tests/e2e-cucumber/features/service_docs.feature create mode 100644 tests/e2e-cucumber/tests/e2e/service_docs_steps.rs diff --git a/Cargo.lock b/Cargo.lock index a46a744a0e..8077abd31f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -434,6 +434,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -504,7 +510,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9d0a013e3d3ee4edd61e779adf117944c08902d375f18630a0c5b8f95659734" dependencies = [ - "base64", + "base64 0.22.1", "bollard-stubs", "bytes", "futures-core", @@ -990,7 +996,7 @@ checksum = "96a87e18d925b19ebe0fd47ea45316abd216d81ec0879c2448c3f9a0e9da62be" dependencies = [ "Inflector", "anyhow", - "base64", + "base64 0.22.1", "clap", "console", "cucumber-codegen", @@ -1479,6 +1485,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1908,7 +1915,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -2105,6 +2112,8 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -2533,6 +2542,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "cfg_aliases 0.2.1", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -3525,9 +3546,10 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", + "futures-util", "http", "http-body", "http-body-util", @@ -3542,12 +3564,14 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.4.2", "web-sys", ] @@ -3557,7 +3581,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -3591,7 +3615,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.5.0", "web-sys", ] @@ -3603,7 +3627,7 @@ checksum = "2d9bae0f2b910cbd4ca2d53513f18aba3b6f6cb6cb0bf295e390d5396f56359d" dependencies = [ "as-any", "async-stream", - "base64", + "base64 0.22.1", "bytes", "eventsource-stream", "fastrand", @@ -3814,7 +3838,7 @@ dependencies = [ "tar", "tempfile", "ureq", - "zip", + "zip 2.4.2", ] [[package]] @@ -3846,6 +3870,7 @@ dependencies = [ "anyhow", "axum", "clap", + "reqwest 0.12.28", "rocm-core", "rocm-engine-protocol", "serde", @@ -3853,6 +3878,8 @@ dependencies = [ "sha2", "tokio", "ureq", + "utoipa", + "utoipa-swagger-ui", "windows-sys 0.61.2", ] @@ -4567,6 +4594,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4731,7 +4769,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bitflags 2.13.0", "fancy-regex", "filedescriptor", @@ -5212,6 +5250,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "typed-path" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ec2755506471f54baf5e4bb5499d8e510cc1caf271270e778a736bd2d7336c" + [[package]] name = "typenum" version = "1.20.1" @@ -5300,7 +5344,7 @@ version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ - "base64", + "base64 0.22.1", "flate2", "log", "once_cell", @@ -5343,6 +5387,51 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "utoipa" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8765fe27aeff71012a3f90fa474cf1df863a7d0dd81ed25de4e63fff2544994f" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d935f1c83fdf8b88f09bbe050d0cea78ea4afee6d7b0317403228c1200b2c8ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1fd2f9615838885c6ff4115fde8a2b58825dc7b4617f78d29c167377a15c92" +dependencies = [ + "base64 0.23.1", + "mime_guess", + "regex", + "serde", + "serde_json", + "utoipa", + "utoipa-swagger-ui-vendored", + "zip 9.0.5", +] + +[[package]] +name = "utoipa-swagger-ui-vendored" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552a64d6a0d251ae28bc021838800943cce60c9ec2e60ff331f54e79f696268f" + [[package]] name = "uuid" version = "1.23.3" @@ -5487,6 +5576,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-streams" version = "0.5.0" @@ -5974,7 +6076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" dependencies = [ "assert-json-diff", - "base64", + "base64 0.22.1", "deadpool", "futures", "http", @@ -6028,7 +6130,7 @@ dependencies = [ "tar", "tempfile", "toml", - "zip", + "zip 2.4.2", ] [[package]] @@ -6224,6 +6326,27 @@ dependencies = [ "zopfli", ] +[[package]] +name = "zip" +version = "9.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "nix 0.31.3", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.21" diff --git a/MANIFEST.md b/MANIFEST.md index 4fdb8bf83f..342eeafc00 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -73,6 +73,7 @@ repository. | axum-core | 0.5.6 | MIT | | backtrace | 0.3.76 | MIT OR Apache-2.0 | | base64 | 0.22.1 | MIT OR Apache-2.0 | +| base64 | 0.23.1 | MIT OR Apache-2.0 | | base64ct | 1.8.3 | Apache-2.0 OR MIT | | bit-set | 0.5.3 | MIT/Apache-2.0 | | bit-vec | 0.6.3 | MIT/Apache-2.0 | @@ -286,6 +287,7 @@ repository. | nanoid | 0.4.0 | MIT | | nix | 0.28.0 | MIT | | nix | 0.29.0 | MIT | +| nix | 0.31.3 | MIT | | nom | 7.1.3 | MIT | | nom | 8.0.0 | MIT | | nom_locate | 5.0.0 | MIT | @@ -458,6 +460,7 @@ repository. | symlink | 0.1.0 | MIT/Apache-2.0 | | syn | 1.0.109 | MIT OR Apache-2.0 | | syn | 2.0.118 | MIT OR Apache-2.0 | +| syn | 3.0.9 | MIT OR Apache-2.0 | | sync_wrapper | 1.0.2 | Apache-2.0 | | synstructure | 0.13.2 | MIT | | synthez | 0.4.0 | BlueOak-1.0.0 | @@ -512,6 +515,7 @@ repository. | tungstenite | 0.28.0 | MIT OR Apache-2.0 | | typed-builder | 0.23.2 | MIT OR Apache-2.0 | | typed-builder-macro | 0.23.2 | MIT OR Apache-2.0 | +| typed-path | 0.12.4 | MIT OR Apache-2.0 | | typenum | 1.20.1 | MIT OR Apache-2.0 | | ucd-trie | 0.1.7 | MIT OR Apache-2.0 | | uds_windows | 1.2.1 | MIT | @@ -529,6 +533,10 @@ repository. | utf-8 | 0.7.6 | MIT OR Apache-2.0 | | utf8_iter | 1.0.4 | Apache-2.0 OR MIT | | utf8parse | 0.2.2 | Apache-2.0 OR MIT | +| utoipa | 6.0.0 | MIT OR Apache-2.0 | +| utoipa-gen | 6.0.1 | MIT OR Apache-2.0 | +| utoipa-swagger-ui | 10.0.2 | MIT OR Apache-2.0 | +| utoipa-swagger-ui-vendored | 0.2.0 | MIT OR Apache-2.0 | | uuid | 1.23.3 | Apache-2.0 OR MIT | | valuable | 0.1.1 | MIT | | version_check | 0.9.5 | MIT/Apache-2.0 | @@ -544,6 +552,7 @@ repository. | wasm-bindgen-macro | 0.2.125 | MIT OR Apache-2.0 | | wasm-bindgen-macro-support | 0.2.125 | MIT OR Apache-2.0 | | wasm-bindgen-shared | 0.2.125 | MIT OR Apache-2.0 | +| wasm-streams | 0.4.2 | MIT OR Apache-2.0 | | wasm-streams | 0.5.0 | MIT OR Apache-2.0 | | web-sys | 0.3.102 | MIT OR Apache-2.0 | | web-time | 1.1.0 | MIT OR Apache-2.0 | @@ -616,6 +625,8 @@ repository. | zerovec | 0.11.6 | Unicode-3.0 | | zerovec-derive | 0.11.3 | Unicode-3.0 | | zip | 2.4.2 | MIT | +| zip | 9.0.5 | MIT | +| zlib-rs | 0.6.8 | Zlib | | zmij | 1.0.21 | MIT | | zopfli | 0.8.3 | Apache-2.0 | | zvariant | 5.12.0 | MIT | @@ -757,6 +768,14 @@ With the prek hooks installed this runs automatically whenever you commit a dependency-graph change, so the notices rarely go stale by hand; the hook skips if cargo-about is absent or not the pinned version. +Assets compiled in that are not crates — today, the Swagger UI bundle behind +the API docs of running model servers — are listed in a "Bundled web assets" +section maintained by hand at the end of `about.hbs`. `cargo-about` cannot see +them, so their licenses bypass the `about.toml` check: when a dependency bump +changes such an asset, update that section from the asset's own upstream +notices. A test in `rocmd` fails when the served Swagger UI version is not the +one the notices name. + CI verifies the committed file is current with `cargo xtask tpn --check`; a new dependency under a license not listed in `about.toml` fails the check until the license is reviewed and added (or the dependency dropped). CI pins the diff --git a/README.md b/README.md index 082a922afb..211ab7c205 100644 --- a/README.md +++ b/README.md @@ -798,7 +798,8 @@ services stop --yes`) or match the original flags. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table -with the status, the full inference endpoint, the API-qualified model name, and a +with the status, the full inference endpoint, a link to the server's API docs +(see API docs below), the API-qualified model name, and a quick smoke test (time to first token and approximate tokens per second). Control returns to your shell with the server still running; manage it later with `rocm services` (below). @@ -884,6 +885,42 @@ started while a `prune` is scanning waits for the prune. `--json` prints the service records verbatim, for scripting and for the remote orchestration below. +#### API docs + +Every running model server gets interactive API docs, served by the background +helper that `rocm serve` starts. Nothing extra needs to be run: once the docs +answer, `rocm serve` prints a `docs` link next to the endpoint, and `rocm +services list` repeats it for each running server (`--json` prints the records +alone, without it): + +``` + endpoint http://127.0.0.1:11435/v1/chat/completions + docs http://127.0.0.1:11430/services//docs/ +``` + +The page lists the routes the server answers (the model list, chat and text +completions, and the engine's health check), with request examples already +filled in with the served model. "Try it out" sends the request through the +docs page to the server, so it works for every engine without enabling +cross-origin access on the server; for a server started with an endpoint key, +enter the key under "Authorize" (it is kept only until the page is closed). +`http://127.0.0.1:11430/` lists every running server. vLLM also documents its +full API at its own `/docs` page, which the docs link to. + +The docs listen on loopback only, accept requests only from pages served on +this machine, and work offline. Change the port, or turn them off, in +`config.json` (`rocm config show` prints where it is): + +```json +{ "api_docs": { "enabled": true, "port": 11430 } } +``` + +The helper reads this when it starts. A `docs` link is printed only once the +docs answer for that server, so there is none when the docs are off, when +another program holds the port, or while the helper that is running was started +by a version of ROCm CLI without API docs or before the port changed; once that +helper exits, the next `rocm serve` starts a new one. + ### Remote machines (preview) Run a model on a different GPU machine and reach it from your own. Both machines diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index 5da3942696..e815b6e3ec 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -1934,6 +1934,217 @@ The following component(s) are licensed under Apache License 2.0: Apache License 2.0 ================================================================================ +The following component(s) are licensed under Apache License 2.0: + - utoipa-gen 6.0.1 + - utoipa-swagger-ui 10.0.2 + - utoipa 6.0.0 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================================ +Apache License 2.0 +================================================================================ + The following component(s) are licensed under Apache License 2.0: - rustls-platform-verifier 0.7.0 - unicode-linebreak 0.1.5 @@ -5757,6 +5968,7 @@ The following component(s) are licensed under Apache License 2.0: - atomic-waker 1.1.2 - backtrace 0.3.76 - base64 0.22.1 + - base64 0.23.1 - bitflags 1.3.2 - bitflags 2.13.0 - blocking 1.6.2 @@ -8203,6 +8415,7 @@ The following component(s) are licensed under Apache License 2.0: - serial2 0.2.37 - simdutf8 0.1.5 - syn 2.0.118 + - syn 3.0.9 - sync_wrapper 1.0.2 - thiserror-impl 1.0.69 - thiserror-impl 2.0.18 @@ -8214,6 +8427,7 @@ The following component(s) are licensed under Apache License 2.0: - unicode-ident 1.0.24 - utf8parse 0.2.2 - vte 0.15.0 + - wasm-streams 0.4.2 - wasm-streams 0.5.0 - winapi-i686-pc-windows-gnu 0.4.0 - winapi-x86_64-pc-windows-gnu 0.4.0 @@ -11839,3 +12053,130 @@ the following restrictions: 3. This notice may not be removed or altered from any source distribution. +================================================================================ +Bundled web assets +================================================================================ + +These components are not Rust crates; they are compiled into rocm-cli by a +crate listed above. Each license's full text appears above. + + - Swagger UI 5.32.6 + Licensed under Apache License 2.0. Compiled in by utoipa-swagger-ui + (through utoipa-swagger-ui-vendored) for the API docs of running model + servers. Its NOTICE file reads: + + swagger-ui + Copyright 2020-2021 SmartBear Software Inc. + + Swagger UI's bundle also contains third-party JavaScript. The notices it + retains, as distributed with Swagger UI 5.32.6 + (swagger-ui-bundle.js.LICENSE.txt in the swagger-ui-dist package; the + standalone preset's notices are a subset of these), read: + + /*! + Copyright (c) 2018 Jed Watson. + Licensed under the MIT License (MIT), see + http://jedwatson.github.io/classnames + */ + + /*! + * @description Recursive object extending + * @author Viacheslav Lotsmanov + * @license MIT + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Viacheslav Lotsmanov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + + /*! + * The buffer module from node.js, for the browser. + * + * @author Feross Aboukhadijeh + * @license MIT + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2021 Joachim Wester + * MIT license + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2022 Joachim Wester + * MIT licensed + */ + + /*! + * repeat-string + * + * Copyright (c) 2014-2015, Jon Schlinkert. + * Licensed under the MIT License. + */ + + /*! @license DOMPurify 3.4.0 | (c) Cure53 and other contributors | Released under the Apache license 2.0 and Mozilla Public License 2.0 | github.com/cure53/DOMPurify/blob/3.4.0/LICENSE */ + + /*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh */ + + /*! js-yaml 4.1.1 https://github.com/nodeca/js-yaml @license MIT */ + + /*! safe-buffer. MIT License. Feross Aboukhadijeh */ + + /** + * @license React + * react-dom.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * react.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * scheduler.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * use-sync-external-store-with-selector.production.js + * + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + diff --git a/about.hbs b/about.hbs index 0781534598..bf281fc8a8 100644 --- a/about.hbs +++ b/about.hbs @@ -30,3 +30,130 @@ The following component(s) are licensed under {{{name}}}: {{{text}}} {{/each}} + +================================================================================ +Bundled web assets +================================================================================ + +These components are not Rust crates; they are compiled into rocm-cli by a +crate listed above. Each license's full text appears above. + + - Swagger UI 5.32.6 + Licensed under Apache License 2.0. Compiled in by utoipa-swagger-ui + (through utoipa-swagger-ui-vendored) for the API docs of running model + servers. Its NOTICE file reads: + + swagger-ui + Copyright 2020-2021 SmartBear Software Inc. + + Swagger UI's bundle also contains third-party JavaScript. The notices it + retains, as distributed with Swagger UI 5.32.6 + (swagger-ui-bundle.js.LICENSE.txt in the swagger-ui-dist package; the + standalone preset's notices are a subset of these), read: + + /*! + Copyright (c) 2018 Jed Watson. + Licensed under the MIT License (MIT), see + http://jedwatson.github.io/classnames + */ + + /*! + * @description Recursive object extending + * @author Viacheslav Lotsmanov + * @license MIT + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Viacheslav Lotsmanov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + + /*! + * The buffer module from node.js, for the browser. + * + * @author Feross Aboukhadijeh + * @license MIT + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2021 Joachim Wester + * MIT license + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2022 Joachim Wester + * MIT licensed + */ + + /*! + * repeat-string + * + * Copyright (c) 2014-2015, Jon Schlinkert. + * Licensed under the MIT License. + */ + + /*! @license DOMPurify 3.4.0 | (c) Cure53 and other contributors | Released under the Apache license 2.0 and Mozilla Public License 2.0 | github.com/cure53/DOMPurify/blob/3.4.0/LICENSE */ + + /*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh */ + + /*! js-yaml 4.1.1 https://github.com/nodeca/js-yaml @license MIT */ + + /*! safe-buffer. MIT License. Feross Aboukhadijeh */ + + /** + * @license React + * react-dom.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * react.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * scheduler.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * use-sync-external-store-with-selector.production.js + * + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ diff --git a/apps/rocm/src/api_docs_link.rs b/apps/rocm/src/api_docs_link.rs new file mode 100644 index 0000000000..d15c30f4e1 --- /dev/null +++ b/apps/rocm/src/api_docs_link.rs @@ -0,0 +1,281 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The API docs link printed for a running model server. +//! +//! The background daemon serves the docs (`rocmd`'s `api_docs` module). A link +//! is only printed once the docs server has answered for that very server, so +//! the CLI never points at a page that is not there: docs turned off, a taken +//! port, or a daemon that has not come up yet all print no link. + +use rocm_core::{AppPaths, RocmCliConfig}; +use std::thread; +use std::time::{Duration, Instant}; + +/// How long one probe of the docs server may take. It is on loopback, so a live +/// server answers in milliseconds and a missing one refuses the connection at once. +const PROBE_TIMEOUT: Duration = Duration::from_millis(500); + +/// How long `rocm serve` waits for a daemon that is still starting. A cold +/// daemon has to parse its config and bind before it answers. +const STARTING_HELPER_WAIT: Duration = Duration::from_secs(5); + +const RETRY_INTERVAL: Duration = Duration::from_millis(200); + +/// How long `rocm serve` should wait for the docs: only a helper that is still +/// starting is worth waiting for. One that was already running answers at once +/// or never will — it may predate API docs, or its port may be taken — and +/// waiting on it would stall every serve. +pub(crate) const fn serve_docs_wait(helper_starting: bool) -> Duration { + if helper_starting { + STARTING_HELPER_WAIT + } else { + Duration::ZERO + } +} + +/// The docs link for `service_id`, if the docs server answers for it within +/// `wait` (one attempt when `wait` is zero). `on_wait` runs between attempts, +/// so a caller can keep its progress indicator moving. +pub(crate) fn service_docs_link( + paths: &AppPaths, + service_id: &str, + wait: Duration, + on_wait: &mut dyn FnMut(), +) -> Option { + let docs = RocmCliConfig::load(paths).ok()?.api_docs; + docs.served_port()?; + let base_url = docs.base_url(); + let deadline = Instant::now() + wait; + loop { + if docs_answer_for(&base_url, service_id) { + return Some(docs.service_docs_url(service_id)); + } + if Instant::now() + RETRY_INTERVAL > deadline { + return None; + } + on_wait(); + thread::sleep(RETRY_INTERVAL); + } +} + +/// Whether the server on the docs port answers with the description of this +/// very service. Checking what it answers, not just that it answers, keeps a +/// link from naming some other program that happens to hold the port. +fn docs_answer_for(base_url: &str, service_id: &str) -> bool { + rocm_core::http_get_text( + base_url, + &format!("/services/{service_id}/openapi.json"), + PROBE_TIMEOUT, + ) + .ok() + .and_then(|body| serde_json::from_str::(&body).ok()) + .is_some_and(|openapi| { + openapi["servers"][0]["url"].as_str() == Some(&format!("/services/{service_id}/api")) + }) +} + +/// [`service_docs_link`] for each of `service_ids`, without waiting. Asks the +/// docs server once whether it is up, so listing many servers while it is down +/// costs one refused connection rather than one per server. +pub(crate) fn service_docs_links<'a>( + paths: &AppPaths, + service_ids: impl IntoIterator, +) -> Vec<(String, String)> { + let Ok(config) = RocmCliConfig::load(paths) else { + return Vec::new(); + }; + if config.api_docs.served_port().is_none() + || rocm_core::http_get_text(&config.api_docs.base_url(), "/health", PROBE_TIMEOUT).is_err() + { + return Vec::new(); + } + service_ids + .into_iter() + .filter_map(|service_id| { + service_docs_link(paths, service_id, Duration::ZERO, &mut || {}) + .map(|link| (service_id.to_owned(), link)) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::test_paths; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + /// A stand-in docs server answering every request with `status` and `body` + /// until the returned flag is set. + fn stand_in_docs_server(status: &'static str, body: String) -> (u16, Arc) { + stand_in_docs_server_after(0, status, body) + } + + /// As [`stand_in_docs_server`], but the first `not_yet` requests are + /// answered 503, as by a docs server still coming up. + fn stand_in_docs_server_after( + not_yet: usize, + status: &'static str, + body: String, + ) -> (u16, Arc) { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + listener.set_nonblocking(true).unwrap(); + let port = listener.local_addr().unwrap().port(); + let stop = Arc::new(AtomicBool::new(false)); + let stop_flag = Arc::clone(&stop); + thread::spawn(move || { + let mut answered = 0; + while !stop_flag.load(Ordering::Relaxed) { + let Ok((mut stream, _)) = listener.accept() else { + thread::sleep(Duration::from_millis(10)); + continue; + }; + stream.set_nonblocking(false).ok(); + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut request = [0_u8; 1024]; + let _ = stream.read(&mut request); + let (status, body) = if answered < not_yet { + ("503 Service Unavailable", "{}") + } else { + (status, body.as_str()) + }; + answered += 1; + let _ = write!( + stream, + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + } + }); + (port, stop) + } + + fn closed_port() -> u16 { + TcpListener::bind(("127.0.0.1", 0)) + .unwrap() + .local_addr() + .unwrap() + .port() + } + + fn paths_with_docs(name: &str, enabled: bool, port: u16) -> (std::path::PathBuf, AppPaths) { + let (root, paths) = test_paths(name); + paths.ensure().unwrap(); + let mut config = RocmCliConfig::default(); + config.api_docs.enabled = enabled; + config.api_docs.port = port; + config.save(&paths).unwrap(); + (root, paths) + } + + fn openapi_for(service_id: &str) -> String { + serde_json::json!({ "servers": [{ "url": format!("/services/{service_id}/api") }] }) + .to_string() + } + + #[test] + fn a_link_is_printed_when_the_docs_describe_this_server() { + let (port, stop) = stand_in_docs_server("200 OK", openapi_for("svc-1")); + let (root, paths) = paths_with_docs("docs-link-answers", true, port); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + Some(format!("http://127.0.0.1:{port}/services/svc-1/docs/")) + ); + assert_eq!( + service_docs_links(&paths, ["svc-1"]), + vec![( + "svc-1".to_owned(), + format!("http://127.0.0.1:{port}/services/svc-1/docs/") + )] + ); + stop.store(true, Ordering::Relaxed); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn no_link_when_the_docs_do_not_know_this_server() { + let (port, stop) = stand_in_docs_server("404 Not Found", "{}".to_owned()); + let (root, paths) = paths_with_docs("docs-link-404", true, port); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + None + ); + assert!(service_docs_links(&paths, ["svc-1"]).is_empty()); + stop.store(true, Ordering::Relaxed); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn no_link_when_another_program_holds_the_docs_port() { + let (port, stop) = stand_in_docs_server("200 OK", openapi_for("some-other-server")); + let (root, paths) = paths_with_docs("docs-link-squatter", true, port); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + None + ); + stop.store(true, Ordering::Relaxed); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn no_link_when_nothing_listens_on_the_docs_port() { + let (root, paths) = paths_with_docs("docs-link-closed", true, closed_port()); + let started = Instant::now(); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + None + ); + assert!(service_docs_links(&paths, ["svc-1", "svc-2"]).is_empty()); + assert!( + started.elapsed() < Duration::from_secs(2), + "a docs server that is not there must not stall the CLI" + ); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn no_link_when_docs_are_turned_off_even_if_the_port_answers() { + let (port, stop) = stand_in_docs_server("200 OK", openapi_for("svc-1")); + let (root, paths) = paths_with_docs("docs-link-disabled", false, port); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + None + ); + assert!(service_docs_links(&paths, ["svc-1"]).is_empty()); + stop.store(true, Ordering::Relaxed); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn no_link_when_the_docs_port_is_zero() { + let (root, paths) = paths_with_docs("docs-link-port-zero", true, 0); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + None + ); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn a_link_waits_for_a_starting_helper_but_not_for_a_running_one() { + assert_eq!(serve_docs_wait(false), Duration::ZERO); + assert!(serve_docs_wait(true) >= Duration::from_secs(1)); + + // A helper that comes up partway through the wait still gets its link, + // and the caller's progress indicator is ticked while it waits. + let (port, stop) = stand_in_docs_server_after(2, "200 OK", openapi_for("svc-1")); + let (root, paths) = paths_with_docs("docs-link-late", true, port); + let mut ticks = 0; + assert_eq!( + service_docs_link(&paths, "svc-1", serve_docs_wait(true), &mut || ticks += 1), + Some(format!("http://127.0.0.1:{port}/services/svc-1/docs/")) + ); + assert_eq!(ticks, 2, "one tick per retry"); + stop.store(true, Ordering::Relaxed); + std::fs::remove_dir_all(root).ok(); + } +} diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index f1c489ce9e..2a4639dcff 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4,6 +4,7 @@ #[cfg(test)] mod advised_commands; +mod api_docs_link; /// Shared archive generator and filesystem oracle for the extraction property /// tests in `comfyui` and `therock`. #[cfg(all(test, unix))] @@ -472,8 +473,8 @@ rocm model --verbose" /// /// Picks an engine and ROCm runtime automatically unless overridden. By default the /// server runs as a managed background service and prints a deployment summary - /// (status, endpoint, model, and smoke-test metrics); use --verbose to stream engine - /// logs in this terminal instead. When streaming, press Ctrl-D to detach and leave the + /// (status, endpoint, API docs link, model, and smoke-test metrics); use --verbose to + /// stream engine logs in this terminal instead. When streaming, press Ctrl-D to detach and leave the /// server running, or Ctrl-C to stop it. Inspect or stop servers later with /// `rocm services`. #[command(after_help = "EXAMPLES:\n \ @@ -624,6 +625,9 @@ rocm logs --search error timeout")] /// - collects GPU thermal/VRAM metrics every 60s for the TUI dashboard /// - listens on a local webhook port for automation events from other /// `rocm` commands + /// - serves API docs for running model servers on 127.0.0.1 (port + /// 11430 by default; `api_docs` in config.json), forwarding their + /// "Try it out" requests to the server /// /// It is normally started on demand by `rocm automations enable` and /// `rocm serve --managed`, so you rarely need to launch it yourself. Run it @@ -6658,6 +6662,13 @@ pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result Result Result<()> { require_api_key, &mut |_elapsed| spinner.tick(), )?; - ensure_background_helper_running_quiet(summary_mode)?; + let helper_starting = ensure_background_helper_running_quiet(summary_mode)?; + let docs_link = crate::api_docs_link::service_docs_link( + &paths, + &report.service_id, + crate::api_docs_link::serve_docs_wait(helper_starting), + &mut || spinner.tick(), + ); // An equivalent service was already running, so nothing was spawned and the // freshly generated key is unused — drop it rather than leave it orphaned in @@ -931,6 +937,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { metrics, api_key: launched_key, notes, + docs_url: docs_link, }; // Same intentional one-time key display as `print_managed_launch_plain` // in service_launch.rs; see its rationale for why the tag below is currently inert. @@ -938,7 +945,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { print!("{}", serve_summary::render_summary(&summary)); } else { spinner.clear(); - print_managed_launch_plain(&report, launched_key.as_deref()); + print_managed_launch_plain(&report, launched_key.as_deref(), docs_link.as_deref()); } return Ok(()); } diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 0f1a7c968b..0c0a498e71 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -64,6 +64,8 @@ pub(crate) struct DeploymentSummary { pub api_key: Option, /// GPU/device warnings folded in from the serve plan. pub notes: Vec, + /// The server's API docs page, when the docs server answered for it. + pub docs_url: Option, } /// Render `Some(ttft)` as a human duration, or `n/a` when the probe did not @@ -127,8 +129,13 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { ("engine", summary.engine.clone()), ], model_rows, + vec![("endpoint", summary.chat_endpoint.clone())], + summary + .docs_url + .iter() + .map(|url| ("docs", url.clone())) + .collect(), vec![ - ("endpoint", summary.chat_endpoint.clone()), ("time to first token", format_ttft(summary.metrics.ttft)), ("throughput (approx)", format_tps(summary.metrics.gen_tps)), ("service", summary.service_id.clone()), @@ -272,9 +279,42 @@ mod tests { }, api_key: None, notes: Vec::new(), + docs_url: None, } } + // Pins only where the row goes. Whether a link is printed at all is + // `api_docs_link`'s tests; that the printed link opens the server's docs is + // the GPU serve canaries' last step (serve-05, serve-07). + #[test] + fn summary_links_the_api_docs_under_the_endpoint_when_they_answered() { + let summary = DeploymentSummary { + docs_url: Some("http://127.0.0.1:11430/services/vllm-qwen-1720000000/docs/".to_owned()), + ..base_summary() + }; + let rendered = render_summary(&summary); + let endpoint_row = rendered + .lines() + .position(|line| line.trim_start().starts_with("endpoint ")) + .expect("endpoint row"); + assert_eq!( + rendered.lines().nth(endpoint_row + 1).map(str::trim), + Some("docs http://127.0.0.1:11430/services/vllm-qwen-1720000000/docs/"), + "the docs row must follow the endpoint row" + ); + } + + #[test] + fn summary_prints_no_docs_row_when_the_docs_did_not_answer() { + let rendered = render_summary(&base_summary()); + assert!( + !rendered + .lines() + .any(|line| line.trim_start().starts_with("docs ")), + "no docs row without an answering docs server" + ); + } + #[test] fn format_ttft_uses_ms_under_a_second_and_seconds_above() { assert_eq!(format_ttft(Some(Duration::from_millis(180))), "180 ms"); diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 4e71cf58b5..f92266c46b 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -629,12 +629,16 @@ pub(crate) fn start_managed_service( pub(crate) fn print_managed_launch_plain( report: &ManagedLaunchReport, endpoint_api_key: Option<&str>, + docs_link: Option<&str>, ) { if report.already_running { println!("managed service already running"); println!(" service_id: {}", report.service_id); println!(" endpoint: {}", report.endpoint_url); println!(" status: {}", report.status); + if let Some(link) = docs_link { + println!(" docs: {link}"); + } println!(" note: existing service detected; no second process spawned"); return; } @@ -664,6 +668,9 @@ pub(crate) fn print_managed_launch_plain( println!(" manifest_path: {}", manifest_path.display()); } println!(" readiness: {}", report.status); + if let Some(link) = docs_link { + println!(" docs: {link}"); + } } /// Render the one-time secure client configuration for a public, authenticated @@ -702,16 +709,21 @@ fn background_helper_already_running(paths: &AppPaths) -> Result { /// itself (`command.spawn()` / `spawn_detached_no_inherit`) is logged rather than /// propagated; setup errors (path discovery, stdio attach) still return `Err`. pub(crate) fn ensure_background_helper_running() -> Result<()> { - ensure_background_helper_running_quiet(false) + ensure_background_helper_running_quiet(false).map(drop) } /// As [`ensure_background_helper_running`], but suppresses the stdout status line /// when `quiet` is set. The interactive `rocm serve` summary path uses `quiet` so /// the daemon-spawn note does not appear above the deployment summary table. -pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> { +/// +/// Returns whether a helper is still starting up — spawned by this call or by a +/// concurrent one — as opposed to one that was already running (or could not be +/// spawned). Only a starting helper is worth waiting for before probing its API +/// docs. +pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result { let paths = AppPaths::discover()?; if background_helper_already_running(&paths)? { - return Ok(()); + return Ok(false); } // The check above and the spawn below are a TOCTOU window: two concurrent @@ -722,7 +734,7 @@ pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> // already-running case lock-free. let _autostart_lock = rocm_core::FileLock::acquire(paths.automation_autostart_lock_path())?; if background_helper_already_running(&paths)? { - return Ok(()); + return Ok(false); } // The lock alone does not close the window: the spawned daemon does not @@ -738,7 +750,7 @@ pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> AUTOSTART_CLAIM_TTL_MS, rocm_core::process_is_running, ) { - return Ok(()); + return Ok(true); } let exe = managed_service_launcher_path() @@ -759,6 +771,7 @@ pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> apply_app_path_env(&mut command, &paths); command.spawn().map(|child| child.id()) }; + let spawned = spawn_result.is_ok(); match spawn_result { Ok(daemon_pid) => { // Record the claim before returning (and thus releasing the lock) so a @@ -780,7 +793,7 @@ pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> } Err(_) => {} } - Ok(()) + Ok(spawned) } /// How long an autostart claim is honoured before it is treated as stale even if @@ -909,6 +922,14 @@ pub(crate) fn run_attached_service( println!(" service_id: {}", report.service_id); println!(" endpoint: {}", report.endpoint_url); println!(" status: {}", report.status); + if let Some(link) = crate::api_docs_link::service_docs_link( + &paths, + &report.service_id, + std::time::Duration::ZERO, + &mut || {}, + ) { + println!(" docs: {link}"); + } println!(" logs: rocm logs {}", report.service_id); println!( " stop: {}", @@ -928,7 +949,7 @@ pub(crate) fn run_attached_service( // server's record, and feeds the dashboard. Match the background ordering // (spawn, then ensure the helper) and keep it quiet so no status line breaks // into the log stream. - ensure_background_helper_running_quiet(true)?; + let helper_starting = ensure_background_helper_running_quiet(true)?; // The resolution detail (model, engine, runtime, GPU, warnings) was already // printed as the "serve plan" block in `serve()`; extend it with the launch @@ -936,6 +957,15 @@ pub(crate) fn run_attached_service( let endpoint = format!("{}/v1", format_http_base_url(host, port)); println!(" service_id: {service_id}"); println!(" endpoint: {endpoint}"); + let docs_link = crate::api_docs_link::service_docs_link( + &paths, + &service_id, + crate::api_docs_link::serve_docs_wait(helper_starting), + &mut || {}, + ); + if let Some(link) = docs_link.as_deref() { + println!(" docs: {link}"); + } if let Some(key) = endpoint_api_key { // Same intentional one-time key display as `print_managed_launch_plain` // above; see its rationale for why the tag below is currently inert. @@ -953,6 +983,9 @@ pub(crate) fn run_attached_service( println!("detached - server still running"); println!(" service_id: {service_id}"); println!(" endpoint: {endpoint}"); + if let Some(link) = docs_link.as_deref() { + println!(" docs: {link}"); + } println!(" list: rocm services"); println!(" logs: rocm logs {service_id}"); println!(" stop: {}", service_hints::service_stop_hint(&service_id)); diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 2f0064e18a..79fa551f78 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -16,6 +16,10 @@ e2e-test-hooks = ["rocm-core/e2e-test-hooks"] [dependencies] anyhow.workspace = true axum.workspace = true +# Forwards the API docs' try-it requests to the model server. Plain HTTP to +# loopback only, so no TLS stack; `stream` relays streamed completions as +# they arrive. +reqwest = { version = "0.12", default-features = false, features = ["stream"] } clap.workspace = true rocm-core = { path = "../../crates/rocm-core" } rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol" } @@ -24,6 +28,11 @@ serde_json.workspace = true sha2 = "0.10" tokio.workspace = true ureq = { version = "2.12", features = ["native-certs"] } +# API docs for running model servers: OpenAPI model + Swagger UI. `vendored` +# compiles the Swagger UI assets in, so the build downloads nothing and the +# page works offline. +utoipa = "6" +utoipa-swagger-ui = { version = "10", default-features = false, features = ["vendored"] } [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } diff --git a/apps/rocmd/src/api_docs.rs b/apps/rocmd/src/api_docs.rs new file mode 100644 index 0000000000..ca57b4a183 --- /dev/null +++ b/apps/rocmd/src/api_docs.rs @@ -0,0 +1,1078 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! API docs for every running model server, served by the daemon on loopback. +//! +//! Of the servers rocm-cli runs, only vLLM generates its own docs; Lemonade and +//! the llama.cpp server behind it serve none. So the daemon describes each +//! running server from the routes rocm-cli knows it serves ([`routes_for`]), +//! renders that with Swagger UI, and forwards the page's "Try it out" requests +//! to the server through `/services//api/...`. Forwarding keeps try-it +//! same-origin, so it works on every engine whatever that engine's own +//! cross-origin settings are; Lemonade, for one, only admits browser requests +//! from its own origin. +//! +//! The server list is read from the service records on every request, so a +//! server started or stopped while the daemon runs needs no notification. + +use anyhow::{Context, Result, bail}; +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::{DefaultBodyLimit, Path, Request, State}; +use axum::http::{HeaderMap, HeaderValue, Method, StatusCode, Uri, header}; +use axum::middleware::{self, Next}; +use axum::response::{Html, IntoResponse, Redirect, Response}; +use axum::routing::{any, get}; +use rocm_core::{AppPaths, DEFAULT_LOCAL_HOST, ManagedServiceRecord}; +use serde_json::{Value, json}; +use std::fmt::Write as _; +use std::net::IpAddr; +use std::sync::Arc; +use std::time::Duration; +use tokio::net::TcpListener; +use utoipa::openapi::path::{HttpMethod, OperationBuilder, PathItem, PathsBuilder}; +use utoipa::openapi::request_body::RequestBodyBuilder; +use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityRequirement, SecurityScheme}; +use utoipa::openapi::{ + ComponentsBuilder, ContentBuilder, InfoBuilder, OpenApi, OpenApiBuilder, Required, + ResponseBuilder, ServerBuilder, +}; + +/// Largest request body the try-it forwarder accepts. Generous enough for an +/// audio upload; the docs server is loopback-only, so this bounds memory, not +/// abuse from elsewhere. +const FORWARD_BODY_LIMIT_BYTES: usize = 64 * 1024 * 1024; + +/// How long the forwarder waits for the model server to accept a connection. +/// There is deliberately no overall timeout: a long completion may stream for +/// minutes. +const FORWARD_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +/// The OpenAPI security scheme name used for servers started with an endpoint +/// key; Swagger UI's "Authorize" box fills it in. +const ENDPOINT_KEY_SCHEME: &str = "endpoint_key"; + +/// A running docs server. It runs on a thread and runtime of its own: the +/// daemon's loop runs watcher checks synchronously on its single-threaded +/// runtime, and a docs page or a streamed try-it answer must not stall for +/// the length of a check. +pub(crate) struct ApiDocsServer { + pub(crate) base_url: String, + shutdown: tokio::sync::oneshot::Sender<()>, + thread: std::thread::JoinHandle<()>, +} + +impl ApiDocsServer { + /// Stop accepting requests and wait for the server thread to finish. + pub(crate) fn stop(self) { + let _ = self.shutdown.send(()); + let _ = self.thread.join(); + } +} + +#[derive(Clone)] +struct DocsState { + paths: Arc, + client: reqwest::Client, +} + +pub(crate) fn start_api_docs_server(paths: AppPaths, port: u16) -> Result { + let listener = std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, port)).with_context(|| { + format!("failed to bind API docs server to {DEFAULT_LOCAL_HOST}:{port}") + })?; + let addr = listener + .local_addr() + .context("failed to read API docs listener address")?; + if !addr.ip().is_loopback() { + bail!("API docs server must bind to loopback; resolved address was {addr}"); + } + listener + .set_nonblocking(true) + .context("failed to configure the API docs listener")?; + let app = router(DocsState { + paths: Arc::new(paths), + client: forwarding_client()?, + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("failed to build the API docs runtime")?; + let (shutdown, shutdown_received) = tokio::sync::oneshot::channel::<()>(); + let thread = std::thread::Builder::new() + .name("rocmd-api-docs".to_owned()) + .spawn(move || { + runtime.block_on(async move { + let served = match TcpListener::from_std(listener) { + Ok(listener) => axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = shutdown_received.await; + }) + .await + .map_err(anyhow::Error::from), + Err(error) => Err(error.into()), + }; + if let Err(error) = served { + eprintln!("API docs server stopped: {error}"); + } + }); + }) + .context("failed to start the API docs thread")?; + Ok(ApiDocsServer { + base_url: format!("http://{addr}"), + shutdown, + thread, + }) +} + +/// The client the forwarder sends try-it requests with. +fn forwarding_client() -> Result { + reqwest::Client::builder() + .connect_timeout(FORWARD_CONNECT_TIMEOUT) + // The model server is on this machine. A proxy configured for the + // user's outbound traffic (`HTTP_PROXY`, `ALL_PROXY`) would carry the + // prompt and the endpoint key off it, or fail the request outright. + .no_proxy() + // A redirect is not followed: the browser gets the model server's own + // status. The routes documented here never redirect. + .redirect(reqwest::redirect::Policy::none()) + .build() + .context("failed to build the API docs forwarding client") +} + +fn router(state: DocsState) -> Router { + Router::new() + .route("/", get(index_page)) + .route("/health", get(health)) + .route("/services/{id}/openapi.json", get(service_openapi_json)) + .route("/services/{id}/docs", get(docs_redirect)) + .route("/services/{id}/docs/", get(docs_index)) + .route("/services/{id}/docs/{*file}", get(docs_file)) + .route( + "/services/{id}/api/{*path}", + any(forward).layer(DefaultBodyLimit::max(FORWARD_BODY_LIMIT_BYTES)), + ) + .layer(middleware::from_fn(require_loopback_host)) + .with_state(state) +} + +/// Refuse requests whose `Host` is not a loopback name, and browser requests +/// sent from a page on another origin. The listener is already loopback-only; +/// the `Host` check closes DNS rebinding, where a web page resolves its own +/// hostname to 127.0.0.1, and the `Origin` check stops any other page the user +/// has open from posting to a model server through the forwarder. Requests +/// that are not from a browser carry no `Origin` and are not affected. +async fn require_loopback_host(request: Request, next: Next) -> Response { + let headers = request.headers(); + let host_allowed = headers + .get(header::HOST) + .and_then(|value| value.to_str().ok()) + .is_some_and(host_header_is_loopback); + let origin_allowed = headers + .get(header::ORIGIN) + .is_none_or(|value| value.to_str().is_ok_and(origin_is_loopback)); + if host_allowed && origin_allowed { + next.run(request).await + } else { + ( + StatusCode::FORBIDDEN, + "API docs answer only on 127.0.0.1 or localhost\n", + ) + .into_response() + } +} + +fn host_header_is_loopback(host: &str) -> bool { + let name = if let Some(rest) = host.strip_prefix('[') { + rest.split(']').next().unwrap_or_default() + } else { + host.rsplit_once(':').map_or(host, |(name, _)| name) + }; + name.eq_ignore_ascii_case("localhost") + || name.parse::().is_ok_and(|ip| ip.is_loopback()) +} + +/// Whether an `Origin` header names a page served from this machine. +fn origin_is_loopback(origin: &str) -> bool { + origin + .strip_prefix("http://") + .or_else(|| origin.strip_prefix("https://")) + .is_some_and(|authority| !authority.contains('/') && host_header_is_loopback(authority)) +} + +async fn health() -> impl IntoResponse { + axum::Json(json!({ "status": "ok", "source": "api_docs", "bind": "loopback_only" })) +} + +/// Every server the record says is up. Errors reading the records read as "no +/// servers": the docs page must never take the daemon down. +fn running_services(paths: &AppPaths) -> Vec { + crate::persistence::load_managed_services(paths) + .unwrap_or_default() + .into_iter() + .filter(service_is_running) + .collect() +} + +fn service_is_running(record: &ManagedServiceRecord) -> bool { + matches!( + record.status.as_str(), + "starting" | "running" | "recovering" | "ready" + ) +} + +/// The running server with this id. Looked up by matching the id against the +/// records rather than by building a path from it, so a crafted id cannot +/// reach outside the services folder. +fn find_running_service(paths: &AppPaths, service_id: &str) -> Option { + running_services(paths) + .into_iter() + .find(|record| record.service_id == service_id) +} + +fn not_running_page(service_id: &str) -> Response { + let id = html_escape(service_id); + ( + StatusCode::NOT_FOUND, + Html(format!( + "Server not running\ +

Server {id} is not running

\ +

Its API docs are available while it runs. \ + Running servers

" + )), + ) + .into_response() +} + +async fn index_page(State(state): State) -> Html { + let services = running_services(&state.paths); + let mut rows = String::new(); + for record in &services { + let id = html_escape(&record.service_id); + let _ = write!( + rows, + "
  • {model} ({engine}, {id}, {endpoint})
  • ", + model = html_escape(&record.canonical_model_id), + engine = html_escape(&record.engine), + endpoint = html_escape(&record.endpoint_url), + ); + } + let body = if services.is_empty() { + "

    No model server is running. Start one with rocm serve.

    ".to_owned() + } else { + format!("
      {rows}
    ") + }; + Html(format!( + "ROCm CLI API docs\ +

    Running model servers

    {body}" + )) +} + +async fn service_openapi_json( + State(state): State, + Path(service_id): Path, +) -> Response { + match find_running_service(&state.paths, &service_id) { + Some(record) => axum::Json(service_openapi(&record)).into_response(), + None => ( + StatusCode::NOT_FOUND, + axum::Json(json!({ + "error": format!("server {service_id} is not running"), + })), + ) + .into_response(), + } +} + +async fn docs_redirect(Path(service_id): Path) -> Redirect { + Redirect::permanent(&format!("/services/{service_id}/docs/")) +} + +async fn docs_index(State(state): State, Path(service_id): Path) -> Response { + serve_swagger_file(&state, &service_id, "") +} + +async fn docs_file( + State(state): State, + Path((service_id, file)): Path<(String, String)>, +) -> Response { + serve_swagger_file(&state, &service_id, &file) +} + +fn serve_swagger_file(state: &DocsState, service_id: &str, file: &str) -> Response { + if find_running_service(&state.paths, service_id).is_none() { + return not_running_page(service_id); + } + let config = Arc::new( + utoipa_swagger_ui::Config::new([format!("/services/{service_id}/openapi.json")]) + // The endpoint key typed into "Authorize" lasts until the page is + // closed; it is not saved in the browser, where it would outlive + // the server it belongs to. + .try_it_out_enabled(true), + ); + match utoipa_swagger_ui::serve(file, config) { + Ok(Some(swagger_file)) => ( + [(header::CONTENT_TYPE, swagger_file.content_type)], + swagger_file.bytes.into_owned(), + ) + .into_response(), + Ok(None) => StatusCode::NOT_FOUND.into_response(), + Err(error) => (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(), + } +} + +/// Relay one try-it request to the model server and stream its answer back. +async fn forward( + State(state): State, + Path((service_id, _decoded_path)): Path<(String, String)>, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let Some(record) = find_running_service(&state.paths, &service_id) else { + return ( + StatusCode::NOT_FOUND, + axum::Json(json!({ "error": format!("server {service_id} is not running") })), + ) + .into_response(); + }; + // The path exactly as the browser sent it. The extracted one is + // percent-decoded: an encoded `?` or `/` in it would turn into a query or + // an extra segment, and the request would reach a different route. + let Some(path) = raw_forwarded_path(uri.path()) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let mut target = format!("{}/{path}", service_base_url(&record)); + if let Some(query) = uri.query() { + target.push('?'); + target.push_str(query); + } + let mut request = state.client.request(method, &target).body(body); + // Only what the routes need: the body's type, the answer the caller + // accepts, and the endpoint key the user typed into "Authorize". Nothing + // the browser adds on its own (cookies, origin) reaches the model server. + for name in [header::CONTENT_TYPE, header::ACCEPT, header::AUTHORIZATION] { + if let Some(value) = headers.get(&name) { + request = request.header(name, value); + } + } + match request.send().await { + Ok(answer) => { + let status = answer.status(); + let content_type = answer.headers().get(header::CONTENT_TYPE).cloned(); + let mut response = Response::new(Body::from_stream(answer.bytes_stream())); + *response.status_mut() = status; + if let Some(content_type) = content_type { + response + .headers_mut() + .insert(header::CONTENT_TYPE, content_type); + } + response + } + Err(error) => ( + StatusCode::BAD_GATEWAY, + [( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + )], + json!({ + "error": format!( + "could not reach server {service_id} at {}: {error}", + service_base_url(&record) + ), + }) + .to_string(), + ) + .into_response(), + } +} + +/// The part of `/services//api/` after `api/`, still percent-encoded. +fn raw_forwarded_path(request_path: &str) -> Option<&str> { + let mut segments = request_path.splitn(5, '/'); + match ( + segments.next(), + segments.next(), + segments.next(), + segments.next(), + segments.next(), + ) { + (Some(""), Some("services"), Some(_), Some("api"), Some(path)) => Some(path), + _ => None, + } +} + +/// Where the forwarder connects. A server bound to every interface is reached +/// on loopback, the same address rocm-cli's own readiness checks use. +fn service_base_url(record: &ManagedServiceRecord) -> String { + let host = match record.host.parse::() { + Ok(ip) if ip.is_unspecified() => DEFAULT_LOCAL_HOST, + _ => record.host.as_str(), + }; + rocm_core::format_http_base_url(host, record.port) +} + +/// One route a model server answers, as the docs describe it. +struct RouteDoc { + method: HttpMethod, + path: &'static str, + summary: &'static str, + description: &'static str, + /// An example request body, built for the server's real model id. + example_body: Option Value>, +} + +/// The routes each engine's server answers: its health route plus +/// [`rocm_engine_protocol::OPENAI_CHAT_ROUTES`], the set its plugin reports as +/// `supported_routes`. The single place the docs learn what a server serves; a +/// new kind of server extends this. +fn routes_for(record: &ManagedServiceRecord) -> Vec { + let health_path = if record.engine == "vllm" { + "/health" + } else { + "/v1/health" + }; + vec![ + RouteDoc { + method: HttpMethod::Get, + path: health_path, + summary: "Check the server is up", + description: "Answers 200 once the server accepts requests.", + example_body: None, + }, + RouteDoc { + method: HttpMethod::Get, + path: "/v1/models", + summary: "List the served models", + description: "OpenAI-compatible model list. The `id` of each entry is the \ + `model` value requests use.", + example_body: None, + }, + RouteDoc { + method: HttpMethod::Post, + path: "/v1/chat/completions", + summary: "Chat completion", + description: "OpenAI-compatible chat completion. Set `stream` to `true` to \ + receive server-sent events as tokens are generated.", + example_body: Some(chat_completion_example), + }, + RouteDoc { + method: HttpMethod::Post, + path: "/v1/completions", + summary: "Text completion", + description: "OpenAI-compatible text completion of a raw prompt.", + example_body: Some(completion_example), + }, + ] +} + +fn chat_completion_example(model: &str) -> Value { + json!({ + "model": model, + "messages": [{ "role": "user", "content": "Say hello in one sentence." }], + "max_tokens": 64, + "stream": false, + }) +} + +fn completion_example(model: &str) -> Value { + json!({ + "model": model, + "prompt": "The capital of France is", + "max_tokens": 16, + }) +} + +/// Where each engine documents more than rocm-cli describes. +fn engine_docs_note(record: &ManagedServiceRecord) -> String { + let base = service_base_url(record); + if record.engine == "vllm" { + format!( + "vLLM also documents every route it serves, beyond the ones listed here, at \ + [{base}/docs]({base}/docs)." + ) + } else { + "On Linux, Lemonade runs chat models with the llama.cpp server, which also answers its \ + own routes; see the \ + [llama.cpp server documentation](https://github.com/ggml-org/llama.cpp/tree/master/tools/server)." + .to_owned() + } +} + +fn service_openapi(record: &ManagedServiceRecord) -> OpenApi { + let model = record.canonical_model_id.as_str(); + let mut paths = PathsBuilder::new(); + for route in routes_for(record) { + let mut operation = OperationBuilder::new() + .summary(Some(route.summary)) + .description(Some(route.description)) + .response( + "200", + ResponseBuilder::new() + .description("The server's answer") + .content("application/json", ContentBuilder::new().build()), + ); + if let Some(example_body) = route.example_body { + operation = operation.request_body(Some( + RequestBodyBuilder::new() + .required(Some(Required::True)) + .content( + "application/json", + ContentBuilder::new() + .example(Some(example_body(model))) + .build(), + ) + .build(), + )); + } + paths = paths.path(route.path, PathItem::new(route.method, operation)); + } + + let description = format!( + "`{model}` served by {engine} at `{endpoint}`.\n\n\ + \"Try it out\" sends requests through this page to the server.\n\n{note}", + engine = record.engine, + endpoint = record.endpoint_url, + note = engine_docs_note(record), + ); + let mut builder = OpenApiBuilder::new() + .info( + InfoBuilder::new() + .title(format!("{model} ({})", record.service_id)) + .version(env!("CARGO_PKG_VERSION")) + .description(Some(description)), + ) + .servers(Some([ServerBuilder::new() + .url(format!("/services/{}/api", record.service_id)) + .description(Some(format!( + "{} through the docs page", + record.endpoint_url + ))) + .build()])) + .paths(paths); + if record.requires_api_key { + builder = builder + .components(Some( + ComponentsBuilder::new() + .security_scheme( + ENDPOINT_KEY_SCHEME, + SecurityScheme::Http( + HttpBuilder::new().scheme(HttpAuthScheme::Bearer).build(), + ), + ) + .build(), + )) + .security(Some([SecurityRequirement::new( + ENDPOINT_KEY_SCHEME, + Vec::::new(), + )])); + } + builder.build() +} + +fn html_escape(text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + match character { + '&' => escaped.push_str("&"), + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + '\'' => escaped.push_str("'"), + other => escaped.push(other), + } + } + escaped +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::temp_app_paths; + use axum::Json; + use axum::routing::post; + use rocm_core::RocmCliConfig; + use std::fs; + use std::sync::Mutex; + + /// Serializes the tests that set proxy variables. Every other test here + /// talks to its servers through [`test_client`], which ignores them. + static PROXY_ENV_TEST_LOCK: Mutex<()> = Mutex::new(()); + + /// The test's own HTTP client: never proxied, so a proxy variable set by + /// the proxy test cannot reroute it. + fn test_client() -> reqwest::Client { + reqwest::Client::builder().no_proxy().build().unwrap() + } + + fn record(paths: &AppPaths, service_id: &str, engine: &str, port: u16) -> ManagedServiceRecord { + let mut record = ManagedServiceRecord::new( + paths, + service_id, + engine, + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record + } + + fn operation_paths(openapi: &Value) -> Vec { + openapi["paths"] + .as_object() + .expect("paths object") + .keys() + .cloned() + .collect() + } + + #[test] + fn chat_server_docs_list_its_routes_with_the_served_model() { + let (root, paths) = temp_app_paths("api-docs-chat-routes"); + for (engine, health) in [("vllm", "/health"), ("lemonade", "/v1/health")] { + let openapi = + serde_json::to_value(service_openapi(&record(&paths, "svc-1", engine, 9))).unwrap(); + let mut listed = operation_paths(&openapi); + listed.sort(); + let mut expected = vec![ + health.to_owned(), + "/v1/chat/completions".to_owned(), + "/v1/completions".to_owned(), + "/v1/models".to_owned(), + ]; + expected.sort(); + assert_eq!(listed, expected, "{engine}"); + assert_eq!( + openapi["paths"]["/v1/chat/completions"]["post"]["requestBody"]["content"]["application/json"] + ["example"]["model"], + "Qwen/Qwen3.5", + "{engine}: try-it must start from the model the server answers to" + ); + assert_eq!( + openapi["servers"][0]["url"], "/services/svc-1/api", + "{engine}" + ); + assert!( + openapi.get("security").is_none(), + "{engine}: loopback needs no key" + ); + } + fs::remove_dir_all(root).ok(); + } + + #[test] + fn documented_routes_are_the_routes_the_engines_report() { + let (root, paths) = temp_app_paths("api-docs-protocol-routes"); + for (engine, health) in [("vllm", "/health"), ("lemonade", "/v1/health")] { + let documented: Vec = routes_for(&record(&paths, "svc-1", engine, 9)) + .iter() + .map(|route| route.path.to_owned()) + .collect(); + assert_eq!( + documented, + rocm_engine_protocol::chat_supported_routes(health), + "{engine}" + ); + } + fs::remove_dir_all(root).ok(); + } + + #[test] + fn forwarded_paths_keep_their_encoding() { + assert_eq!( + raw_forwarded_path("/services/svc-1/api/v1/models/Qwen%2FQwen3.5%3Fx"), + Some("v1/models/Qwen%2FQwen3.5%3Fx") + ); + assert_eq!(raw_forwarded_path("/services/svc-1/api/"), Some("")); + assert_eq!(raw_forwarded_path("/services/svc-1/docs/x"), None); + } + + #[test] + fn only_pages_from_this_machine_may_send_browser_requests() { + for origin in [ + "http://127.0.0.1:11430", + "http://localhost:11430", + "http://[::1]:11430", + ] { + assert!(origin_is_loopback(origin), "{origin}"); + } + for origin in [ + "https://evil.example", + "http://127.0.0.1.evil.example", + "null", + "", + ] { + assert!(!origin_is_loopback(origin), "{origin}"); + } + } + + #[test] + fn the_bundled_swagger_ui_is_the_version_named_in_the_notices() { + let config = Arc::new(utoipa_swagger_ui::Config::new(["/openapi.json"])); + let bundle = utoipa_swagger_ui::serve("swagger-ui-bundle.js", config) + .unwrap() + .expect("the vendored Swagger UI ships its bundle"); + let bundle = String::from_utf8_lossy(&bundle.bytes); + let marker = "PACKAGE_VERSION:\""; + let start = bundle.find(marker).expect("the bundle states its version") + marker.len(); + let version: String = bundle[start..].chars().take_while(|c| *c != '"').collect(); + // Read, not `include_str!`: the file is not ASCII, and xtask's output + // contract rejects non-ASCII data compiled into a binary. + let notices = fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../THIRD_PARTY_NOTICES.txt" + )) + .expect("THIRD_PARTY_NOTICES.txt at the repo root"); + assert!( + notices.contains(&format!(" - Swagger UI {version} ")), + "THIRD_PARTY_NOTICES.txt must name the Swagger UI {version} that rocmd serves; \ + update the bundled-assets section of about.hbs and run `cargo xtask tpn`" + ); + } + + #[test] + fn docs_stay_off_without_stopping_the_daemon() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-off"); + paths.ensure()?; + let mut state = crate::service::build_runtime_state(&RocmCliConfig::default(), true); + + let mut disabled = RocmCliConfig::default(); + disabled.api_docs.enabled = false; + assert!(crate::service::start_api_docs(&paths, &disabled, &mut state)?.is_none()); + + let mut port_zero = RocmCliConfig::default(); + port_zero.api_docs.port = 0; + assert!(crate::service::start_api_docs(&paths, &port_zero, &mut state)?.is_none()); + + let taken = std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, 0))?; + let mut on_taken_port = RocmCliConfig::default(); + on_taken_port.api_docs.port = taken.local_addr()?.port(); + assert!( + crate::service::start_api_docs(&paths, &on_taken_port, &mut state)?.is_none(), + "a taken port leaves docs off instead of failing the daemon" + ); + let events = fs::read_to_string(paths.automation_events_path())?; + assert!(events.contains("api_docs_unavailable"), "{events}"); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[test] + fn keyed_server_docs_ask_for_the_endpoint_key() { + let (root, paths) = temp_app_paths("api-docs-keyed"); + let mut keyed = record(&paths, "svc-1", "vllm", 9); + keyed.requires_api_key = true; + let openapi = serde_json::to_value(service_openapi(&keyed)).unwrap(); + assert_eq!( + openapi["components"]["securitySchemes"][ENDPOINT_KEY_SCHEME]["scheme"], + "bearer" + ); + assert!(openapi["security"][0].get(ENDPOINT_KEY_SCHEME).is_some()); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn engine_note_links_vllm_docs_and_llama_cpp_routes() { + let (root, paths) = temp_app_paths("api-docs-note"); + assert!( + engine_docs_note(&record(&paths, "a", "vllm", 8000)) + .contains("http://127.0.0.1:8000/docs") + ); + assert!(engine_docs_note(&record(&paths, "b", "lemonade", 8000)).contains("llama.cpp")); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn a_server_bound_to_every_interface_is_reached_on_loopback() { + let (root, paths) = temp_app_paths("api-docs-wildcard"); + let mut wildcard = record(&paths, "a", "vllm", 8000); + wildcard.host = "0.0.0.0".to_owned(); + assert_eq!(service_base_url(&wildcard), "http://127.0.0.1:8000"); + wildcard.host = "::".to_owned(); + assert_eq!(service_base_url(&wildcard), "http://127.0.0.1:8000"); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn only_loopback_host_names_are_accepted() { + for host in [ + "127.0.0.1:11430", + "localhost:11430", + "LOCALHOST", + "[::1]:11430", + "127.0.0.1", + ] { + assert!(host_header_is_loopback(host), "{host}"); + } + for host in [ + "evil.example:11430", + "192.168.1.5:11430", + "[2001:db8::1]:11430", + "", + ] { + assert!(!host_header_is_loopback(host), "{host}"); + } + } + + #[test] + fn html_escape_neutralises_markup() { + assert_eq!( + html_escape("'&'"), + "<a href="x">'&'</a>" + ); + } + + /// A stand-in model server that echoes what reached it, so the test can + /// see exactly what the forwarder passed on. + async fn start_echo_model_server() -> u16 { + let listener = TcpListener::bind((DEFAULT_LOCAL_HOST, 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let app = Router::new().route( + "/v1/chat/completions", + post(|headers: HeaderMap, uri: Uri, body: Bytes| async move { + Json(json!({ + "query": uri.query(), + "authorization": headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + "cookie": headers.get(header::COOKIE).and_then(|value| value.to_str().ok()), + "body": String::from_utf8_lossy(&body), + })) + }), + ); + tokio::spawn(async move { axum::serve(listener, app).await.ok() }); + port + } + + #[tokio::test(flavor = "multi_thread")] + async fn docs_server_describes_and_forwards_to_a_running_server() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-live"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let model_port = start_echo_model_server().await; + record(&paths, "svc-live", "vllm", model_port).write()?; + let mut stopped = record(&paths, "svc-stopped", "vllm", model_port); + stopped.status = "stopped".to_owned(); + stopped.write()?; + + let server = start_api_docs_server(paths.clone(), 0)?; + let base = server.base_url.clone(); + let client = test_client(); + + let index = client.get(format!("{base}/")).send().await?.text().await?; + assert!(index.contains("/services/svc-live/docs/"), "{index}"); + assert!(!index.contains("svc-stopped"), "{index}"); + + let page = client + .get(format!("{base}/services/svc-live/docs/")) + .send() + .await?; + assert_eq!(page.status(), StatusCode::OK); + assert!(page.text().await?.contains("swagger-ui")); + let initializer = client + .get(format!( + "{base}/services/svc-live/docs/swagger-initializer.js" + )) + .send() + .await? + .text() + .await?; + assert!( + initializer.contains("/services/svc-live/openapi.json"), + "Swagger UI must load this server's description: {initializer}" + ); + + let openapi: Value = client + .get(format!("{base}/services/svc-live/openapi.json")) + .send() + .await? + .text() + .await? + .parse()?; + assert_eq!(openapi["servers"][0]["url"], "/services/svc-live/api"); + + let forwarded: Value = client + .post(format!( + "{base}/services/svc-live/api/v1/chat/completions?trace=1" + )) + .header(header::AUTHORIZATION, "Bearer typed-in-docs") + .header(header::COOKIE, "session=browser-only") + .header(header::CONTENT_TYPE, "application/json") + .body(r#"{"model":"Qwen/Qwen3.5"}"#) + .send() + .await? + .text() + .await? + .parse()?; + assert_eq!(forwarded["body"], r#"{"model":"Qwen/Qwen3.5"}"#); + assert_eq!(forwarded["query"], "trace=1"); + assert_eq!(forwarded["authorization"], "Bearer typed-in-docs"); + assert_eq!( + forwarded["cookie"], + Value::Null, + "browser cookies stay in the browser" + ); + + for path in ["docs/", "openapi.json", "api/v1/models"] { + let response = client + .get(format!("{base}/services/svc-stopped/{path}")) + .send() + .await?; + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{path}"); + } + let stopped_page = client + .get(format!("{base}/services/svc-stopped/docs/")) + .send() + .await? + .text() + .await?; + assert!(stopped_page.contains("is not running"), "{stopped_page}"); + + let rebound = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::HOST, "attacker.example") + .body("{}") + .send() + .await?; + assert_eq!(rebound.status(), StatusCode::FORBIDDEN, "DNS rebinding"); + let cross_site = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::ORIGIN, "https://some-site.example") + .header(header::CONTENT_TYPE, "text/plain") + .body("{}") + .send() + .await?; + assert_eq!( + cross_site.status(), + StatusCode::FORBIDDEN, + "cross-site post" + ); + let same_origin = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::ORIGIN, base.as_str()) + .body("{}") + .send() + .await?; + assert_eq!(same_origin.status(), StatusCode::OK, "the docs page itself"); + + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// Restores a proxy variable when the test ends, even by panicking. + struct RestoredProxyVar(&'static str, Option); + + impl Drop for RestoredProxyVar { + #[allow(unsafe_code)] // std::env::set_var/remove_var are unsafe in edition 2024 + fn drop(&mut self) { + // SAFETY: still inside PROXY_ENV_TEST_LOCK, which the test holds + // for longer than this guard lives. + unsafe { + match &self.1 { + Some(value) => std::env::set_var(self.0, value), + None => std::env::remove_var(self.0), + } + } + } + } + + // A plain test driving its own runtime, so the std lock guarding the + // process environment is never held across an `.await`. + #[allow(unsafe_code)] // std::env::set_var is unsafe in edition 2024 + #[test] + fn try_it_reaches_the_server_even_with_a_proxy_configured() -> Result<()> { + let _lock = PROXY_ENV_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + let (root, paths) = temp_app_paths("api-docs-proxy"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let model_port = runtime.block_on(start_echo_model_server()); + record(&paths, "svc-live", "vllm", model_port).write()?; + // A proxy that is not there: a request sent to it fails. + let dead_proxy = format!( + "http://127.0.0.1:{}", + std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, 0))? + .local_addr()? + .port() + ); + // Every original value is saved before any is changed: on Windows + // `HTTP_PROXY` and `http_proxy` name one variable, so saving one after + // setting the other would save the dead proxy as the "original". + let proxied = ["HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy"]; + let unproxied = ["NO_PROXY", "no_proxy"]; + let _restore: Vec = proxied + .iter() + .chain(&unproxied) + .map(|key| RestoredProxyVar(key, std::env::var_os(key))) + .collect(); + for key in proxied { + // SAFETY: PROXY_ENV_TEST_LOCK is held; `_restore` puts the old + // values back before it is released. + unsafe { std::env::set_var(key, &dead_proxy) }; + } + for key in unproxied { + // SAFETY: as above. + unsafe { std::env::remove_var(key) }; + } + + let server = start_api_docs_server(paths, 0)?; + let status = runtime.block_on(async { + test_client() + .post(format!( + "{}/services/svc-live/api/v1/chat/completions", + server.base_url + )) + .body("{}") + .send() + .await + .map(|response| response.status()) + })?; + assert_eq!(status, StatusCode::OK, "try-it went to the proxy"); + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn forwarding_to_a_dead_server_reports_a_bad_gateway() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-dead"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + // Bind and drop to get a port nothing listens on. + let dead_port = std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, 0))? + .local_addr()? + .port(); + record(&paths, "svc-dead", "lemonade", dead_port).write()?; + let server = start_api_docs_server(paths.clone(), 0)?; + let response = test_client() + .get(format!( + "{}/services/svc-dead/api/v1/models", + server.base_url + )) + .send() + .await?; + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert!( + response + .text() + .await? + .contains("could not reach server svc-dead") + ); + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } +} diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index 3333461903..d442e53138 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -4,6 +4,7 @@ #![allow(clippy::items_after_test_module)] +mod api_docs; mod cli; mod common; mod mcp; diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 1a8c3335e9..b35ea8d185 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -351,6 +351,40 @@ fn terminate_recorded_pid(identity: &rocm_core::ProcessIdentity) -> rocm_core::T ) } +/// Start the API docs server for running model servers, unless `config.json` +/// turns it off. A taken port leaves docs off and is recorded as an +/// `api_docs_unavailable` automation event; the daemon's own work carries on +/// either way. +pub(crate) fn start_api_docs( + paths: &AppPaths, + config: &RocmCliConfig, + state: &mut AutomationRuntimeState, +) -> Result> { + let Some(port) = config.api_docs.served_port() else { + println!(" api_docs: disabled"); + return Ok(None); + }; + match crate::api_docs::start_api_docs_server(paths.clone(), port) { + Ok(server) => { + println!(" api_docs: {}", server.base_url); + Ok(Some(server)) + } + Err(error) => { + println!(" api_docs: off ({error:#})"); + crate::persistence::record_event( + paths, + state, + "rocmd", + "warn", + "api_docs_unavailable", + &format!("API docs for running model servers are off: {error:#}"), + None, + )?; + Ok(None) + } + } +} + pub(crate) async fn run_daemon( paths: &AppPaths, automations_enabled: bool, @@ -415,6 +449,10 @@ pub(crate) async fn run_daemon( } paths.ensure()?; + // Before the first state write: that write is what tells `rocm serve` a + // helper is already running, and a serve that reads it then probes the + // docs once, without waiting for them. + let api_docs = start_api_docs(paths, &config, &mut state)?; state.write(paths)?; crate::persistence::record_event( paths, @@ -517,6 +555,9 @@ pub(crate) async fn run_daemon( if let Some(task) = local_webhook_task { task.abort(); } + if let Some(server) = api_docs { + server.stop(); + } Ok(()) } @@ -862,7 +903,7 @@ fn engine_serve_http_args( args } -fn build_runtime_state( +pub(crate) fn build_runtime_state( config: &RocmCliConfig, automations_enabled: bool, ) -> AutomationRuntimeState { diff --git a/crates/rocm-core/src/config.rs b/crates/rocm-core/src/config.rs index 860ddcf0e7..1c5cfa1371 100644 --- a/crates/rocm-core/src/config.rs +++ b/crates/rocm-core/src/config.rs @@ -69,6 +69,66 @@ pub struct AutomationsConfig { pub watchers: BTreeMap, } +/// Port the background daemon serves API docs on when `config.json` sets none. +/// Kept clear of `DEFAULT_LOCAL_PORT` and the ports above it that consecutive +/// `rocm serve` launches pick. +pub const DEFAULT_API_DOCS_PORT: u16 = 11_430; + +/// API docs for running model servers, served by the background daemon on +/// loopback. Described in: README.md's API docs section, docs/testing.md and +/// docs/manual-testing.md. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ApiDocsConfig { + #[serde(default = "default_api_docs_enabled")] + pub enabled: bool, + #[serde(default = "default_api_docs_port")] + pub port: u16, +} + +impl Default for ApiDocsConfig { + fn default() -> Self { + Self { + enabled: default_api_docs_enabled(), + port: default_api_docs_port(), + } + } +} + +const fn default_api_docs_enabled() -> bool { + true +} + +const fn default_api_docs_port() -> u16 { + DEFAULT_API_DOCS_PORT +} + +impl ApiDocsConfig { + /// The port the docs are served on, or `None` when they are off: turned off + /// in `config.json`, or set to port 0, which would bind a random port no + /// link could name. + #[must_use] + pub const fn served_port(&self) -> Option { + if self.enabled && self.port != 0 { + Some(self.port) + } else { + None + } + } + + /// The docs server's root: `GET` lists every running server. + #[must_use] + pub fn base_url(&self) -> String { + crate::format_http_base_url(crate::DEFAULT_LOCAL_HOST, self.port) + } + + /// Swagger UI for one running server. The trailing slash is part of the + /// address: the page loads its assets relative to it. + #[must_use] + pub fn service_docs_url(&self, service_id: &str) -> String { + format!("{}/services/{service_id}/docs/", self.base_url()) + } +} + #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ProviderUserConfig { #[serde(default)] @@ -210,6 +270,8 @@ pub struct RocmCliConfig { pub engines: BTreeMap, #[serde(default)] pub automations: AutomationsConfig, + #[serde(default)] + pub api_docs: ApiDocsConfig, /// rocm-dash telemetry/dashboard knobs. Nested as a sub-config /// so it never collides with the rocm-cli `telemetry` analytics policy on /// rebase. Every field defaults, so the section is fully optional. diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index ec4163dc25..1708a899ed 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -56,10 +56,11 @@ pub use automation::{ }; pub use binary_paths::{daemon_binary_path, engine_binary_path, sibling_binary_path}; pub use config::{ - AutomationsConfig, DashboardConfig, DashboardDaemonConfig, DashboardTuiConfig, - EngineUserConfig, ManagedToolConfig, PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, - PermissionsConfig, ProviderUserConfig, RocmCliConfig, SetupConfig, TELEMETRY_MODE_LOCAL, - TELEMETRY_MODE_OFF, TelemetryConfig, WatcherMode, WatcherUserConfig, + ApiDocsConfig, AutomationsConfig, DEFAULT_API_DOCS_PORT, DashboardConfig, + DashboardDaemonConfig, DashboardTuiConfig, EngineUserConfig, ManagedToolConfig, + PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, PermissionsConfig, ProviderUserConfig, + RocmCliConfig, SetupConfig, TELEMETRY_MODE_LOCAL, TELEMETRY_MODE_OFF, TelemetryConfig, + WatcherMode, WatcherUserConfig, }; pub use crypto::{ generate_rsa_signing_keypair, sign_rsa_pkcs1_sha256_signature, diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index fbca3104fc..00bdecc673 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -600,6 +600,22 @@ impl HealthcheckResponse { } } +/// The OpenAI-compatible routes every chat server answers. +/// +/// Each engine reports them, after its own health route, in +/// [`EndpointResponse::supported_routes`]; the daemon's API docs are tested +/// against [`chat_supported_routes`] so the two lists agree. +pub const OPENAI_CHAT_ROUTES: [&str; 3] = ["/v1/models", "/v1/chat/completions", "/v1/completions"]; + +/// `supported_routes` for a chat server whose health route is `health_route`. +#[must_use] +pub fn chat_supported_routes(health_route: &str) -> Vec { + std::iter::once(health_route) + .chain(OPENAI_CHAT_ROUTES) + .map(str::to_owned) + .collect() +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EndpointResponse { pub endpoint_url: String, diff --git a/docs/architecture.md b/docs/architecture.md index d2e1c90348..66ec831807 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -29,6 +29,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every [`main.rs`](../apps/rocm/src/main.rs) itself is **partially modularized** — Phase 6a (ROCMAI-82) extracted the driver-install/engines/serve clusters, Phase 6b (ROCMAI-91) extracted the managed-process-launch cluster, Phase 6c (ROCMAI-89) extracted the `runtimes` command/uninstall/rollback/SDK-install-finalization clusters, Phase 6d (ROCMAI-88) extracted the chat/assistant and MCP tool-bridge cluster below across three PRs; the rest of the split is planned across several more PRs, one cluster at a time (see EAI-7768). +- [`api_docs_link.rs`](../apps/rocm/src/api_docs_link.rs) — full domain extraction: the API docs link printed by `rocm serve` and `rocm services list`, probed against the docs server before it is printed. The docs themselves are served by `rocmd` ([`api_docs.rs`](../apps/rocmd/src/api_docs.rs)). - [`automations.rs`](../apps/rocm/src/automations.rs) — mechanical relocation, dispatch-adjacent, no owned types. - [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) — a further-extracted variant of full domain extraction: it owns its clap command enum (`BootstrapCommand`) and dispatch function too, rather than leaving them in [`main.rs`](../apps/rocm/src/main.rs). - [`chat.rs`](../apps/rocm/src/chat.rs) — full domain extraction (Phase 6d, ROCMAI-88), mirrors [`providers.rs`](../apps/rocm/src/providers.rs). Owns the chat/MCP-bridge cluster's types (`ChatRocmCommandAction`, `CommandCapture`, `DeterministicModelRecipe`, `DeterministicServiceRow`, `DeterministicEngineRow`, and others — all module-private except `ChatToolApprovalRequest`, reached by `dash_seam.rs` as `chat_tool_approval_request`'s return type), its validation/approval/execution logic (`validate_chat_tool_call` and the `validate_chat_*_tool_call` family), deterministic tool-output summarization (including `render_services_tool_result_text`), and the MCP tool-bridge entry points (`run_internal_mcp_call` and its helpers). [`main.rs`](../apps/rocm/src/main.rs), [`dash.rs`](../apps/rocm/src/dash.rs), and [`dash_seam.rs`](../apps/rocm/src/dash_seam.rs) call into it in production; [`comfyui.rs`](../apps/rocm/src/comfyui.rs) and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) reference it only from their own `#[cfg(test)]` modules (`deterministic_comfyui_tool_summary` and `rocm_chat_tool_requested_args` respectively) — every one of these five files' references still goes through the `chat::` module boundary. As a child module of the crate root (`mod chat;` in `main.rs`), Rust's module-privacy rule already makes most of `main.rs`'s private helpers visible to `chat.rs` without any visibility change, when `chat.rs` calls into `main.rs`. The other direction needs real widening: `chat.rs`'s own functions reached from those five files — `run_internal_mcp_call` (a production call from `main.rs`), `rocm_chat_tool_requested_args` (reached only by `runtime_install_finalize.rs`'s own test module), among others — stay `pub(crate)`; the third PR's visibility audit tightened every other function to module-private, leaving 20 of `chat.rs`'s 123 functions `pub(crate)`, each with a confirmed external caller, production or test-only. The cluster's regression tests live in `chat.rs`'s own `mod tests`, moved out of `main.rs`'s shared `mod tests` block across the second and third PRs; two `main.rs` tests that reference `chat::` only incidentally — structural checks on `main.rs`'s own `dispatch()` source (`command_chat_honors_chat_mock_and_keeps_prompt_passthrough`, `command_chat_reads_prompt_from_piped_stdin`) — plus `serve_negative_sampling_space_form_reaches_range_validator`, a shared clap value-parser wiring test living alongside its sibling command-parsing tests, stay in `main.rs`. @@ -59,6 +60,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every Modularized (ROCMAI-83, Phase 5 of EAI-7768's sequencing): +- [`api_docs.rs`](../apps/rocmd/src/api_docs.rs) — added after that split, as its own module from the start: the API docs server `run_daemon` starts on a thread of its own: one OpenAPI description per running model server built with `utoipa` from a route catalogue, Swagger UI from `utoipa-swagger-ui`, and the forwarder that carries "Try it out" requests to the server. - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, `SandboxToolArg`/`SandboxToolPolicy`, the bridge-snapshot diagnostic printer, and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. - [`common.rs`](../apps/rocmd/src/common.rs) — helpers shared across ≥2 of the other clusters: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, `CommandCapture` and the shared `rocm`-subprocess capture helpers the sandbox and MCP clusters both call (the bounded-execution half of that plumbing lives in [`process.rs`](../crates/rocm-core/src/process.rs) rather than here), and small arg/healthcheck/endpoint-key utilities. A helper earns a place here only once a second still-inline cluster calls it directly; a helper with exactly one caller stays next to that caller until its own cluster's extraction PR, even if it is conceptually similar to something that did move. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the `rocm`-subprocess capture/argv-building helpers behind the MCP tools. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index f29df54862..8982937830 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -614,6 +614,27 @@ Expected result: On an 80x24 terminal, repeat the steps up to the dialog: the card shows the outcome (report ready or refusal) without scrolling. +### 4b. API docs + +With a model served by `rocm serve ` (any engine): + +Expected result: + +- The deployment summary has a `docs` row under `endpoint`, and `rocm services + list` shows the same `docs:` link for that server. +- Opening the link in a browser shows the server's routes. Under + `POST /v1/chat/completions`, "Try it out" then "Execute" returns a chat + completion from the served model; the example body already names that model. +- The page loads with the network disconnected (the docs need no CDN). +- For a server started with `--require-api-key` (an endpoint key), "Execute" + is refused until the key is entered under "Authorize". +- After `rocm services stop --yes`, reloading the page says the + server is not running, and `rocm services list` prints no docs link. +- With `"api_docs": { "enabled": false }` in `config.json` and the helper + restarted, `rocm serve` prints no docs link. +- With `HTTP_PROXY` set to an address nothing listens on before the helper + starts, "Try it out" still reaches the server. + ## 5. ComfyUI Verification ComfyUI is managed as an app surface. It should start a local web server and diff --git a/docs/testing.md b/docs/testing.md index 30b1486e47..d07e9b8a7e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -300,6 +300,40 @@ export ROCM_TEST_APK_REPOS="--repository http://dl-cdn.alpinelinux.org/alpine/v3 --repository http://dl-cdn.alpinelinux.org/alpine/v3.20/community" ``` +## API docs + +The API docs server lives in `rocmd` +([api_docs.rs](../apps/rocmd/src/api_docs.rs)); the link the CLI prints is +built in [api_docs_link.rs](../apps/rocm/src/api_docs_link.rs). Unit tests run a +real docs server against a stand-in model server, covering the route catalogue, +the forwarder (what it passes on and what it drops, and that a configured proxy +does not divert it), stopped and unreachable servers, the loopback `Host` and +`Origin` guard, and a taken or zero docs port. The CLI's tests cover when a +link is and is not printed: + +```bash +cargo test -p rocmd api_docs +cargo test -p rocm --bin rocm api_docs_link +cargo test -p rocm --bin rocm serve_summary +``` + +[service_docs.feature](../tests/e2e-cucumber/features/service_docs.feature) +starts the real `rocm daemon` in the scenario's isolated tree, on a port of its +own set in that tree's `config.json`, and drives the CLI and the docs over HTTP. +`service-docs-03` stops the server with `rocm services stop`, so it is Linux +only (`@requires-os:linux`): + +```bash +cargo xtask e2e -- -n "service-docs-" +``` + +The link `rocm serve` itself prints, and a try-it request through it to a real +engine, are checked by the GPU serve canaries `serve-05` (vLLM) and `serve-07` +(Lemonade), which only run on GPU lanes. On Linux Lemonade lanes `serve-07` is +an expected failure for an unrelated known issue, so there a broken docs link +would not turn the lane red: the Lemonade check is effective on the Windows +Lemonade lanes, and `serve-05` on the vLLM lanes. + ## CI test selection On pull requests, CI does not test the whole workspace. The `test` job runs diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 7f8fac2934..30c4e304f2 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -745,12 +745,7 @@ fn endpoint_response(request: EndpointRequest) -> Result { Ok(EndpointResponse { endpoint_url, api_style: "openai".to_owned(), - supported_routes: vec![ - "/v1/health".to_owned(), - "/v1/models".to_owned(), - "/v1/chat/completions".to_owned(), - "/v1/completions".to_owned(), - ], + supported_routes: rocm_engine_protocol::chat_supported_routes("/v1/health"), }) } diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index 6ad87f9379..836a9c9073 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -84,12 +84,7 @@ pub(crate) fn endpoint_response(request: EndpointRequest) -> Result, + /// A `rocm daemon` a scenario started so its API docs answer; killed at + /// teardown, before the isolated tree it reads is removed. + pub background_helper: Option, + /// The API docs link a `service-docs` scenario read from the CLI. + pub docs_link: Option, + /// What a successful real `rocm serve` printed, for steps that check the + /// launch output after the server came up. + pub serve_stdout: Option, } /// One scenario's resolved expectation plus the identity needed to report it. @@ -323,6 +332,9 @@ impl Default for E2eWorld { skill_symptom: None, recorded_process: None, dash_report_refused: None, + background_helper: None, + docs_link: None, + serve_stdout: None, } } } @@ -653,6 +665,10 @@ impl Drop for E2eWorld { let _ = process.kill(); let _ = process.wait(); } + if let Some(mut helper) = self.background_helper.take() { + let _ = helper.kill(); + let _ = helper.wait(); + } if let Some(mock) = self.mock.take() { mock.stop(); } diff --git a/tests/e2e-cucumber/tests/e2e/service_docs_steps.rs b/tests/e2e-cucumber/tests/e2e/service_docs_steps.rs new file mode 100644 index 0000000000..2b59b38615 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/service_docs_steps.rs @@ -0,0 +1,312 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `service_docs.feature`: the API docs the background helper serves +//! for every running model server. +//! +//! The helper is the real `rocm daemon`, started in the scenario's isolated +//! tree on a port of its own (set in that tree's `config.json`), so parallel +//! scenarios and a helper already running on the host never share docs. + +use std::time::{Duration, Instant}; + +use cucumber::{given, then, when}; +use serde_json::Value; + +use crate::E2eWorld; + +/// How long a cold `rocm daemon` gets to start answering. +const HELPER_START_TIMEOUT: Duration = Duration::from_secs(30); + +fn free_port() -> u16 { + std::net::TcpListener::bind(("127.0.0.1", 0)) + .and_then(|listener| listener.local_addr()) + .map(|addr| addr.port()) + .expect("failed to reserve a loopback port") +} + +/// The docs link `rocm services list` printed for the one listed server. +fn docs_link_in(listing: &str) -> Option { + listing + .lines() + .find_map(|line| line.trim().strip_prefix("docs: ")) + .map(str::to_owned) +} + +async fn get(url: &str) -> reqwest::Response { + reqwest::get(url) + .await + .unwrap_or_else(|error| panic!("GET {url} failed: {error}")) +} + +/// The OpenAPI description a docs page loads, found the way the page finds it: +/// from the Swagger UI initializer served next to the page. +async fn openapi_behind(docs_link: &str) -> (String, Value) { + let initializer = get(&format!("{docs_link}swagger-initializer.js")) + .await + .text() + .await + .expect("docs initializer body"); + let start = initializer + .find("/services/") + .unwrap_or_else(|| panic!("the docs page names no description:\n{initializer}")); + let path: String = initializer[start..] + .chars() + .take_while(|character| *character != '"') + .collect(); + let origin = docs_link + .split("/services/") + .next() + .expect("docs link has an origin"); + let openapi = get(&format!("{origin}{path}")) + .await + .json::() + .await + .expect("the docs description is JSON"); + (origin.to_owned(), openapi) +} + +/// Point this scenario's API docs at a free port of their own, in the isolated +/// `config.json`, and return it. A helper `rocm serve` leaves behind in another +/// scenario keeps its own port, so it can never answer for this one. Calling it +/// again keeps the port already chosen: a serve retried within one scenario +/// reuses the helper the first attempt started, which listens there. +pub(crate) fn give_api_docs_a_port_of_their_own(world: &E2eWorld) -> u16 { + let paths = world.app_paths(); + std::fs::create_dir_all(&paths.config_dir).expect("failed to create config dir"); + let config_path = paths.config_path(); + let mut config: Value = std::fs::read(&config_path) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .unwrap_or_else(|| serde_json::json!({})); + if let Some(port) = config["api_docs"]["port"] + .as_u64() + .and_then(|port| u16::try_from(port).ok()) + { + return port; + } + let port = free_port(); + config["api_docs"] = serde_json::json!({ "enabled": true, "port": port }); + std::fs::write(&config_path, serde_json::to_vec_pretty(&config).unwrap()) + .expect("failed to write config.json"); + port +} + +/// The docs link in `rocm serve`'s output: the deployment summary's `docs` row +/// or the plain output's `docs:` line. +fn docs_link_in_serve_output(stdout: &str) -> Option { + stdout.lines().find_map(|line| { + let rest = line.trim().strip_prefix("docs")?; + let link = rest.trim_start_matches(':').trim(); + link.starts_with("http://").then(|| link.to_owned()) + }) +} + +// ── Given ────────────────────────────────────────────────────────── + +#[given("the background helper is not running")] +async fn background_helper_not_running(world: &mut E2eWorld) { + // Nothing listens on this scenario's docs port: the state a host is in + // when no helper is up, which a helper running on the host itself cannot + // disturb. + give_api_docs_a_port_of_their_own(world); +} + +#[given("the background helper is running")] +async fn background_helper_running(world: &mut E2eWorld) { + let port = give_api_docs_a_port_of_their_own(world); + let paths = world.app_paths(); + + // The helper's own output names why docs did not come up (`api_docs: off + // (...)`), so it is kept for the failure message rather than discarded. + let log_path = paths.data_dir.join("background-helper.log"); + std::fs::create_dir_all(&paths.data_dir).expect("failed to create data dir"); + let log = std::fs::File::create(&log_path).expect("failed to create helper log"); + let mut command = std::process::Command::new(crate::rocm_binary()); + command + .arg("daemon") + .stdin(std::process::Stdio::null()) + .stdout(log.try_clone().expect("failed to share helper log")) + .stderr(log); + world.isolate_cmd(&mut command); + world.background_helper = Some(command.spawn().expect("failed to start rocm daemon")); + + let health = format!("http://127.0.0.1:{port}/health"); + let deadline = Instant::now() + HELPER_START_TIMEOUT; + while !reqwest::get(&health) + .await + .is_ok_and(|response| response.status().is_success()) + { + let exited = world + .background_helper + .as_mut() + .and_then(|helper| helper.try_wait().ok().flatten()); + let output = || std::fs::read_to_string(&log_path).unwrap_or_default(); + assert!( + exited.is_none(), + "the background helper exited ({exited:?}) before its API docs answered:\n{}", + output() + ); + assert!( + Instant::now() < deadline, + "the background helper's API docs did not answer on port {port} within \ + {HELPER_START_TIMEOUT:?}; its output:\n{}", + output() + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +#[given("the user has opened the server's API docs")] +async fn user_opened_docs(world: &mut E2eWorld) { + let listing = crate::run_rocm_ok(world, &["services", "list"]); + let link = docs_link_in(&listing) + .unwrap_or_else(|| panic!("premise: a running server has a docs link:\n{listing}")); + let page = get(&link).await; + assert!(page.status().is_success(), "premise: the docs open"); + world.docs_link = Some(link); +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("the user tries the chat completions route from the server's API docs")] +async fn user_tries_chat_route(world: &mut E2eWorld) { + let listing = crate::run_rocm_ok(world, &["services", "list"]); + let link = docs_link_in(&listing) + .unwrap_or_else(|| panic!("no API docs link in the services list:\n{listing}")); + let (origin, openapi) = openapi_behind(&link).await; + // What "Try it out" sends: the route's example body, to the server the + // description names. + let server = openapi["servers"][0]["url"] + .as_str() + .expect("the docs name a server to send requests to"); + let body = openapi["paths"]["/v1/chat/completions"]["post"]["requestBody"]["content"] + ["application/json"]["example"] + .clone(); + let response = reqwest::Client::new() + .post(format!("{origin}{server}/v1/chat/completions")) + .json(&body) + .send() + .await + .expect("try-it request failed to send"); + let status = response.status(); + let answer = response.text().await.unwrap_or_default(); + world.cli_output = Some(format!("{status}\n{answer}")); +} + +#[when("the user stops the managed model")] +async fn user_stops_managed_model(world: &mut E2eWorld) { + crate::run_rocm_ok(world, &["services", "stop", "e2e-mock", "--yes"]); +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then("the running server has an API docs link")] +async fn running_server_has_docs_link(world: &mut E2eWorld) { + let listing = world.cli_output.as_deref().expect("no services listing"); + let link = docs_link_in(listing) + .unwrap_or_else(|| panic!("no API docs link in the services list:\n{listing}")); + assert!( + link.ends_with("/services/e2e-mock/docs/"), + "the link names a different server: {link}" + ); + world.docs_link = Some(link); +} + +#[then("the link opens API docs listing the chat routes with the server's model")] +async fn link_opens_chat_docs(world: &mut E2eWorld) { + let link = world.docs_link.clone().expect("no docs link"); + let page = get(&link).await; + assert!(page.status().is_success(), "the docs page did not open"); + assert!( + page.text().await.unwrap_or_default().contains("swagger-ui"), + "the link did not open the API docs page" + ); + let (_, openapi) = openapi_behind(&link).await; + for route in ["/v1/models", "/v1/chat/completions", "/v1/completions"] { + assert!( + openapi["paths"].get(route).is_some(), + "the docs do not list {route}:\n{openapi:#}" + ); + } + let model = world.model_name.as_deref().expect("no model name"); + assert_eq!( + openapi["paths"]["/v1/chat/completions"]["post"]["requestBody"]["content"]["application/json"] + ["example"]["model"], + model, + "the docs' example does not use the server's model" + ); +} + +#[then("the model server answers the request")] +async fn model_server_answers(world: &mut E2eWorld) { + let answer = world.cli_output.as_deref().expect("no try-it answer"); + let (status, body) = answer.split_once('\n').unwrap_or((answer, "")); + assert!( + status.starts_with("200"), + "try-it was not answered: {answer}" + ); + let body: Value = serde_json::from_str(body) + .unwrap_or_else(|_| panic!("the answer is not the server's JSON: {answer}")); + assert!( + body["choices"][0]["message"]["content"].is_string(), + "the answer is not a chat completion: {body:#}" + ); +} + +#[then("the server's API docs say it is not running")] +async fn docs_say_not_running(world: &mut E2eWorld) { + let link = world.docs_link.clone().expect("no docs link"); + let page = get(&link).await; + assert_eq!(page.status(), reqwest::StatusCode::NOT_FOUND); + let text = page.text().await.unwrap_or_default(); + assert!(text.contains("is not running"), "{text}"); +} + +#[then("the services list still shows the stopped server, without an API docs link")] +async fn stopped_server_listed_without_docs_link(world: &mut E2eWorld) { + // `--all`: the default list hides stopped servers, so it could not show a + // docs line for one whatever the CLI did. + let listing = crate::run_rocm_ok(world, &["services", "list", "--all"]); + assert!( + listing.contains("- e2e-mock") && listing.contains("status: stopped"), + "premise: the stopped server is still listed:\n{listing}" + ); + assert_eq!(docs_link_in(&listing), None, "{listing}"); +} + +#[then("the running server has no API docs link")] +async fn running_server_has_no_docs_link(world: &mut E2eWorld) { + let listing = world.cli_output.as_deref().expect("no services listing"); + assert!( + listing.contains("- e2e-mock"), + "premise: the running server is listed:\n{listing}" + ); + assert_eq!(docs_link_in(listing), None, "{listing}"); +} + +#[then("the API docs link printed when it started opens the server's API docs")] +async fn serve_printed_docs_link_opens(world: &mut E2eWorld) { + let stdout = world.serve_stdout.as_deref().expect("no serve output"); + let link = docs_link_in_serve_output(stdout) + .unwrap_or_else(|| panic!("`rocm serve` printed no API docs link:\n{stdout}")); + let (origin, openapi) = openapi_behind(&link).await; + for route in ["/v1/models", "/v1/chat/completions", "/v1/completions"] { + assert!( + openapi["paths"].get(route).is_some(), + "the docs do not list {route}:\n{openapi:#}" + ); + } + // The docs' "Try it out" reaches the real engine: list its models through + // the docs page. + let server = openapi["servers"][0]["url"] + .as_str() + .expect("the docs name a server to send requests to"); + let models = get(&format!("{origin}{server}/v1/models")).await; + assert!( + models.status().is_success(), + "try-it through the docs did not reach the engine: {}", + models.status() + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 7ce18b326b..0374dec286 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -145,10 +145,12 @@ fn serve_failure_evidence( async fn serve_and_wait(world: &mut E2eWorld, args: &[&str], model: &str, ready_substr: &str) { let timeout_secs = serve_timeout_for(world); let device_state = ensure_serve_port_free().await; + crate::e2e::service_docs_steps::give_api_docs_a_port_of_their_own(world); let (stdout, stderr, rc) = crate::run_rocm(world, args); if rc == 0 && model_is_ready(&MODELS_URL, Some(ready_substr), timeout_secs).await { world.endpoint = Some(SERVE_BASE_URL.to_string()); world.model_name = Some(model.to_string()); + world.serve_stdout = Some(stdout); return; } let invocation = format!("`rocm {}`", args.join(" ")); @@ -544,11 +546,13 @@ async fn serve_gpu_model(world: &mut E2eWorld) { for attempt in 1..=attempts { made = attempt; let device_state = ensure_serve_port_free().await; + crate::e2e::service_docs_steps::give_api_docs_a_port_of_their_own(world); let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", model, "--engine", engine, "--managed"]); if rc == 0 && model_is_ready(&MODELS_URL, Some(ready_substr), timeout_secs).await { world.endpoint = Some(SERVE_BASE_URL.to_string()); world.model_name = Some(model.to_string()); + world.serve_stdout = Some(stdout); return; } // `serve_failure_evidence` reads and copies out the log BEFORE it stops diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index c40fc7ec0a..c280b996da 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -48,6 +48,7 @@ const FEATURE_KEYS: &[(&str, &str)] = &[ // sharing one key would collide on every index (`runtime-01` in both). ("runtime_lifecycle.feature", "runtime-lifecycle"), ("runtime_setup.feature", "runtime"), + ("service_docs.feature", "service-docs"), ("service_record_cleanup.feature", "service-cleanup"), ("service_stop.feature", "service-stop"), ("therock_next_generation.feature", "therock-next"), From 71277cc2561d6f795a04e2dbb06dce9ef31eae6d Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 20 Oct 2026 09:11:12 +0000 Subject: [PATCH 2/7] feat(serve): serve speech-to-text models through Lemonade Every served model was assumed to be a chat model: the protocol had no notion of what a model does, readiness was proven with a chat request, and the serve summary always printed /v1/chat/completions. Add a model task (chat or transcription) to catalog recipes, service records and the engine protocol. Every new field defaults to chat, so records, state files and external engine plugins written before this change keep their behaviour, and a plugin that does not advertise tasks is treated as chat-only. - rocm serve refuses a task the selected engine does not advertise before the GPU gate, naming an engine that can serve it. - The Lemonade adapter serves transcription models through its router with an explicit whispercpp backend (ROCm, else Vulkan, never CPU) and proves a test transcription before reporting ready. Lemonade's per-model device label is not trusted: 11.5.1 reports ROCm Whisper as cpu, so the loaded backend is verified instead, as for chat. - Readiness probes send a transcription for a speech-to-text service rather than a chat request, whose 404 would read as serving. - The task survives restarts from both rocm and rocmd. - vLLM advertises chat only and refuses other tasks itself. Signed-off-by: Roman Inflianskas --- apps/rocm/src/main.rs | 19 + apps/rocm/src/providers.rs | 3 +- apps/rocm/src/serve_cmd.rs | 99 ++++- apps/rocm/src/serve_summary.rs | 56 ++- apps/rocm/src/service_launch.rs | 10 + apps/rocmd/src/service.rs | 10 + crates/rocm-core/src/lib.rs | 11 +- crates/rocm-core/src/model_catalog.rs | 32 ++ crates/rocm-core/src/model_task.rs | 123 ++++++ crates/rocm-core/src/net.rs | 256 ++++++++++++- crates/rocm-core/src/service_record.rs | 6 + crates/rocm-engine-protocol/src/lib.rs | 102 ++++- engines/lemonade/src/backend_alignment.rs | 41 +- engines/lemonade/src/lib.rs | 42 +- engines/lemonade/src/process.rs | 42 +- engines/lemonade/src/speech.rs | 443 ++++++++++++++++++++++ engines/lemonade/src/state.rs | 43 ++- engines/vllm/src/lib.rs | 3 + engines/vllm/src/process.rs | 56 +++ engines/vllm/src/state.rs | 1 + 20 files changed, 1346 insertions(+), 52 deletions(-) create mode 100644 crates/rocm-core/src/model_task.rs create mode 100644 engines/lemonade/src/speech.rs diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 2a4639dcff..76e6eccc5e 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -327,6 +327,8 @@ enum Command { log_path: Option, #[arg(long)] engine_recipe_json: Option, + #[arg(long, default_value = "chat")] + task: String, }, #[command(name = "__engine-stdio", hide = true)] EngineStdio { engine: String }, @@ -2237,6 +2239,7 @@ fn dispatch(cli: Cli) -> Result<()> { state_path, log_path, engine_recipe_json, + task, }) => run_builtin_engine_serve_http( &engine, service_id, @@ -2250,6 +2253,7 @@ fn dispatch(cli: Cli) -> Result<()> { state_path, log_path, parse_engine_recipe_json_arg(engine_recipe_json)?, + rocm_core::ModelTask::parse(&task)?, ), Some(Command::EngineStdio { engine }) => run_builtin_engine_stdio(&engine), Some(Command::InternalStatus) => { @@ -7180,6 +7184,7 @@ fn restart_child_argv(record: &ManagedServiceRecord) -> Result> { recipe.as_ref(), &record.engine_state_path, Some(&record.log_path), + record.task, ) } @@ -10289,8 +10294,15 @@ fn run_builtin_engine_serve_http( state_path: PathBuf, log_path: Option, engine_recipe: Option, + task: rocm_core::ModelTask, ) -> Result<()> { let parsed_policy = parse_device_policy(Some(device_policy))?; + if engine == "vllm" && !task.is_chat() { + bail!( + "the vLLM engine in rocm-cli serves chat models only; it cannot serve a {} model", + task.display_name() + ); + } match engine { "lemonade" => rocm_engine_lemonade::builtin_serve_http( service_id, @@ -10304,6 +10316,7 @@ fn run_builtin_engine_serve_http( state_path, log_path, engine_recipe, + task, ), "vllm" => rocm_engine_vllm::builtin_serve_http( service_id, @@ -10336,6 +10349,7 @@ fn builtin_engine_serve_http_args( engine_recipe: Option<&EngineRecipeHint>, state_path: &Path, log_path: Option<&Path>, + task: rocm_core::ModelTask, ) -> Result> { let mut args = vec![ "__engine-serve-http".to_owned(), @@ -10361,6 +10375,10 @@ fn builtin_engine_serve_http_args( if let Some(log_path) = log_path { args.extend(["--log-path".to_owned(), log_path.display().to_string()]); } + // Omitted for chat so a chat launch keeps the argv it always had. + if !task.is_chat() { + args.extend(["--task".to_owned(), task.as_str().to_owned()]); + } Ok(args) } @@ -19503,6 +19521,7 @@ pub(crate) mod tests { tool_calling: false, quantized_models: "vllm-supported".to_owned(), reasoning_parser: false, + tasks: vec![rocm_core::ModelTask::Chat], }, notes: Vec::new(), }; diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index e655914113..888f6f0b8a 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -2920,7 +2920,8 @@ mod tests { engine: "vllm".to_owned(), requested_model: "Qwen/Qwen3.5".to_owned(), api_model: "Qwen/Qwen3.5".to_owned(), - chat_endpoint: "http://127.0.0.1:8000/v1/chat/completions".to_owned(), + endpoint: "http://127.0.0.1:8000/v1/chat/completions".to_owned(), + task: rocm_core::ModelTask::Chat, service_id: "svc-ready".to_owned(), status: "ready".to_owned(), already_running: false, diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index f764cccb99..a243478127 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -23,16 +23,16 @@ use std::fmt::Write as _; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, ModelRecipeRecord, RocmCliConfig, default_engine_for_platform, detect_host_gpu_summary, generate_service_id, preferred_serve_engine_for_host_gpu_summary, resolve_model_recipe, }; use rocm_engine_protocol::{ - DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EngineMethod, EngineRecipeEndpointHint, - EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, GpuSelection, ResolveModelRequest, - ResolveModelResponse, + DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EngineCapabilities, EngineMethod, + EngineRecipeEndpointHint, EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, + GpuSelection, ResolveModelRequest, ResolveModelResponse, }; use crate::DevicePolicyArg; @@ -577,6 +577,25 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { host_gpu_summary.as_ref(), ); let selected_engine = serve_engine.engine.clone(); + // A model outside the catalog keeps the behaviour it always had: it is + // served, and probed, as a chat model. + let task = shared_recipe + .as_ref() + .map(ModelRecipeRecord::model_task) + .transpose()? + .unwrap_or_default(); + // Before the GPU gate and any engine preparation, so a speech-to-text model + // sent to an engine that cannot serve one fails with the reason on every + // host — not with a GPU or runtime error that hides it. + if !task.is_chat() { + ensure_engine_serves_task( + &paths, + &selected_engine, + task, + &model, + shared_recipe.as_ref(), + )?; + } // Fail closed: a public bind must be authenticated, but Windows managed // Lemonade cannot receive the key (see `ensure_public_bind_engine_supported`), // so refuse rather than launch an open public server. @@ -745,6 +764,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { device_policy: Some(device_policy), recipe_override: None, engine_recipe, + task, }, )?; // Serialize GPU auto-selection with the managed-service claim: the busy-GPU @@ -779,6 +799,9 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { println!(" resolved model: {}", resolve.canonical_model_id); println!(" engine: {selected_engine}"); println!("{}", serve_engine_selection_line(&serve_engine)); + if !task.is_chat() { + println!(" task: {}", task.display_name()); + } println!(" host: {host}"); println!(" port: {port}"); if let Some(runtime_id) = resolved_selection.runtime_id.as_deref() { @@ -872,6 +895,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { endpoint_auth.as_deref(), launch_lock, require_api_key, + task, &mut |_elapsed| spinner.tick(), )?; let helper_starting = ensure_background_helper_running_quiet(summary_mode)?; @@ -904,7 +928,13 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Best-effort inference smoke test, on by default (opt out with // `--no-smoke-test`). Only meaningful for a freshly-ready server we // just launched; skipped when metrics could not be shown anyway. - let metrics = if !no_smoke_test && !report.already_running && report.status == "ready" { + // The smoke test streams a chat completion; a speech-to-text server + // was already proven by a test transcription before it reported ready. + let metrics = if task.is_chat() + && !no_smoke_test + && !report.already_running + && report.status == "ready" + { spinner.set_label("Running smoke test…"); // The local provider resolves the endpoint key from the per-service // 0600 key file by service id, so the smoke test authenticates @@ -930,7 +960,8 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { engine: selected_engine.clone(), requested_model: model, api_model: resolve.canonical_model_id, - chat_endpoint: format!("{}/chat/completions", report.endpoint_url), + endpoint: format!("{}{}", report.endpoint_url, task.api_route()), + task, service_id: report.service_id.clone(), status: report.status.clone(), already_running: report.already_running, @@ -946,6 +977,14 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } else { spinner.clear(); print_managed_launch_plain(&report, launched_key.as_deref(), docs_link.as_deref()); + if !task.is_chat() { + println!( + " {} endpoint: {}{}", + task.display_name(), + report.endpoint_url, + task.api_route() + ); + } } return Ok(()); } @@ -963,9 +1002,57 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { endpoint_auth.as_deref(), launch_lock, require_api_key, + task, ) } +/// Refuse to hand `engine` a model whose task it does not advertise. +/// +/// The engine's own capabilities decide, so an external plugin that predates +/// tasks — and therefore reports chat only — is refused here too, rather than +/// being launched with a model it would treat as a chat model. +fn ensure_engine_serves_task( + paths: &AppPaths, + engine: &str, + task: rocm_core::ModelTask, + model: &str, + recipe: Option<&ModelRecipeRecord>, +) -> Result<()> { + let capabilities = engine_request::<_, EngineCapabilities>( + Some(paths), + engine, + EngineMethod::Capabilities, + &serde_json::json!({}), + ) + .with_context(|| format!("failed to read the capabilities of engine `{engine}`"))?; + if capabilities.serves(task) { + return Ok(()); + } + bail!("{}", unsupported_task_message(engine, task, model, recipe)) +} + +fn unsupported_task_message( + engine: &str, + task: rocm_core::ModelTask, + model: &str, + recipe: Option<&ModelRecipeRecord>, +) -> String { + let kind = task.display_name(); + let alternative = recipe.and_then(|recipe| { + recipe + .preferred_engines + .iter() + .find(|candidate| !candidate.eq_ignore_ascii_case(engine)) + }); + match alternative { + Some(alternative) => format!( + "`{model}` is a {kind} model, which the {engine} engine cannot serve; \ + serve it with `rocm serve {model} --engine {alternative}`" + ), + None => format!("`{model}` is a {kind} model, which the {engine} engine cannot serve"), + } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 0c0a498e71..83ca6bf8eb 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -50,8 +50,12 @@ pub(crate) struct DeploymentSummary { /// Canonical / API-qualified model id clients pass as `"model"`. May differ /// from `requested_model` when a recipe alias resolves to a canonical id. pub api_model: String, - /// Full chat-completions endpoint, e.g. `http://127.0.0.1:1337/v1/chat/completions`. - pub chat_endpoint: String, + /// Full endpoint for the model's task, e.g. + /// `http://127.0.0.1:1337/v1/chat/completions` for a chat model or + /// `http://127.0.0.1:1337/v1/audio/transcriptions` for a speech-to-text one. + pub endpoint: String, + /// What the model does; decides which rows apply. + pub task: rocm_core::ModelTask, pub service_id: String, /// `"ready"`, `"starting"`, or an existing service's status. pub status: String, @@ -123,21 +127,33 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { ] }; + // Token timings describe text generation. A speech-to-text server has none + // to report, and two permanent `n/a` rows would read as a failed + // measurement; it names its task instead, since its endpoint is not the one + // chat clients expect. + let task_rows: Vec<(&str, String)> = if summary.task.is_chat() { + vec![ + ("time to first token", format_ttft(summary.metrics.ttft)), + ("throughput (approx)", format_tps(summary.metrics.gen_tps)), + ] + } else { + vec![("task", summary.task.display_name().to_owned())] + }; + let rows: Vec<(&str, String)> = [ vec![ ("status", summary.status.clone()), ("engine", summary.engine.clone()), ], model_rows, - vec![("endpoint", summary.chat_endpoint.clone())], + vec![("endpoint", summary.endpoint.clone())], summary .docs_url .iter() .map(|url| ("docs", url.clone())) .collect(), + task_rows, vec![ - ("time to first token", format_ttft(summary.metrics.ttft)), - ("throughput (approx)", format_tps(summary.metrics.gen_tps)), ("service", summary.service_id.clone()), ( "stop", @@ -269,7 +285,8 @@ mod tests { engine: "vllm".to_owned(), requested_model: "Qwen/Qwen2.5-7B-Instruct".to_owned(), api_model: "Qwen/Qwen2.5-7B-Instruct".to_owned(), - chat_endpoint: "http://127.0.0.1:1337/v1/chat/completions".to_owned(), + endpoint: "http://127.0.0.1:1337/v1/chat/completions".to_owned(), + task: rocm_core::ModelTask::Chat, service_id: "vllm-qwen-1720000000".to_owned(), status: "ready".to_owned(), already_running: false, @@ -350,6 +367,31 @@ mod tests { assert!(rendered.contains("rocm services stop vllm-qwen-1720000000 --yes")); } + #[test] + fn speech_to_text_summary_names_its_task_and_omits_token_timings() { + // `serve()` builds `endpoint` from the same task value it launched the + // engine with (`ModelTask::api_route`), and skips the chat smoke test + // for this task; the e2e scenario + // `@id:serve-speech-to-text-model-prints-its-transcription-endpoint` + // proves the printed route is the one the running server answers. + let summary = DeploymentSummary { + engine: "lemonade".to_owned(), + requested_model: "whisper".to_owned(), + api_model: "Whisper-Large-v3-Turbo".to_owned(), + endpoint: "http://127.0.0.1:1337/v1/audio/transcriptions".to_owned(), + task: rocm_core::ModelTask::Transcription, + metrics: SmokeMetrics::default(), + ..base_summary() + }; + let rendered = render_summary(&summary); + assert!(rendered.contains("http://127.0.0.1:1337/v1/audio/transcriptions")); + assert!(rendered.contains("task"), "{rendered}"); + assert!(rendered.contains("speech-to-text"), "{rendered}"); + assert!(!rendered.contains("time to first token"), "{rendered}"); + assert!(!rendered.contains("throughput"), "{rendered}"); + assert!(!rendered.contains("chat/completions"), "{rendered}"); + } + #[test] fn summary_structure_is_identical_across_engines() { // The same fields render in the same order regardless of engine; only the @@ -373,7 +415,7 @@ mod tests { lemonade.engine = "lemonade".to_owned(); lemonade.requested_model = "Qwen3-0.6B-GGUF".to_owned(); lemonade.api_model = "Qwen3-0.6B-GGUF".to_owned(); - lemonade.chat_endpoint = "http://127.0.0.1:8000/v1/chat/completions".to_owned(); + lemonade.endpoint = "http://127.0.0.1:8000/v1/chat/completions".to_owned(); lemonade.metrics = SmokeMetrics::default(); // engine reported no usage assert_eq!( diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index f92266c46b..e035e4cdbe 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -327,6 +327,7 @@ fn spawn_managed_engine_child( env_id: Option<&str>, engine_recipe: Option<&EngineRecipeHint>, require_api_key: bool, + task: rocm_core::ModelTask, ) -> Result { paths.ensure()?; fs::create_dir_all(paths.services_dir())?; @@ -415,6 +416,7 @@ fn spawn_managed_engine_child( ); record.gpu_indices = gpu_indices.to_vec(); record.engine_recipe_json = requested_recipe_json; + record.task = task; // The flag the user actually passed, carried through rather than re-derived. // // Deriving it from key-file presence looked equivalent and was not: @@ -451,6 +453,7 @@ fn spawn_managed_engine_child( engine_recipe, &record.engine_state_path, Some(&record.log_path), + task, )?; let engine_envs_root = env_root_for_service(paths, engine, runtime_id, env_id)?; // Hand the child the *path* to the endpoint key file (public bind only) via the @@ -552,6 +555,7 @@ pub(crate) fn start_managed_service( endpoint_api_key: Option<&str>, launch_lock: rocm_core::FileLock, require_api_key: bool, + task: rocm_core::ModelTask, on_wait_tick: &mut dyn FnMut(Duration), ) -> Result { let paths = AppPaths::discover()?; @@ -569,6 +573,7 @@ pub(crate) fn start_managed_service( env_id, engine_recipe, require_api_key, + task, )? { ManagedSpawn::AlreadyRunning(report) => return Ok(report), ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), @@ -890,6 +895,7 @@ pub(crate) fn run_attached_service( endpoint_api_key: Option<&str>, launch_lock: rocm_core::FileLock, require_api_key: bool, + task: rocm_core::ModelTask, ) -> Result<()> { let paths = AppPaths::discover()?; @@ -907,6 +913,7 @@ pub(crate) fn run_attached_service( env_id, resolve.engine_recipe.as_ref(), require_api_key, + task, )?; // The claiming record is persisted (or an existing service was found), so the // selected GPU is now visible to concurrent auto-selection. Release the launch @@ -1284,6 +1291,7 @@ mod tests { None, Some(&requested_recipe), false, + rocm_core::ModelTask::Chat, ); let _ = fs::remove_dir_all(root); @@ -1368,6 +1376,7 @@ mod tests { None, // The demand that used to be silently discarded. true, + rocm_core::ModelTask::Chat, ); let _ = fs::remove_dir_all(root); @@ -1453,6 +1462,7 @@ mod tests { None, None, true, + rocm_core::ModelTask::Chat, ); let _ = fs::remove_dir_all(root); diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index b35ea8d185..bcd1dec2bc 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -748,6 +748,7 @@ pub(crate) fn supervise_service( env_id.as_deref(), engine_recipe_json.as_deref(), &record.engine_state_path, + record.task, )) .stdin(Stdio::null()) .stdout(Stdio::from(log_file)) @@ -877,6 +878,7 @@ fn engine_serve_http_args( env_id: Option<&str>, engine_recipe_json: Option<&str>, state_path: &Path, + task: rocm_core::ModelTask, ) -> Vec { let mut args = vec![ "__engine-serve-http".to_owned(), @@ -900,6 +902,11 @@ fn engine_serve_http_args( engine_recipe_json, )); args.extend(["--state-path".to_owned(), state_path.display().to_string()]); + // Omitted for chat so a chat restart keeps the argv it always had; a + // speech-to-text service must come back as one. + if !task.is_chat() { + args.extend(["--task".to_owned(), task.as_str().to_owned()]); + } args } @@ -1366,6 +1373,7 @@ mod tests { Some("env-1"), Some(engine_recipe_json), Path::new("state.json"), + rocm_core::ModelTask::Chat, ); assert!( @@ -1397,6 +1405,7 @@ mod tests { None, None, Path::new("state.json"), + rocm_core::ModelTask::Chat, ); assert!( @@ -1416,6 +1425,7 @@ mod tests { None, None, Path::new("state.json"), + rocm_core::ModelTask::Chat, ); assert!(!auto.iter().any(|arg| arg == "--gpu")); } diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 1708a899ed..b89e632143 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -27,6 +27,7 @@ pub mod log_tail; pub mod managed_runtime; pub mod model_catalog; pub mod model_readiness; +pub mod model_task; pub mod net; pub mod openmpi; pub mod os_release; @@ -123,18 +124,20 @@ pub use model_catalog::{ normalize_model_ref, platform_matches_gfx_family, resolve_builtin_model_recipe, resolve_model_recipe, resolve_model_recipe_artifact, verify_model_recipe_index_signature, }; +pub use model_task::ModelTask; pub use net::{ Backoff, DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, DownloadRequest, EndpointReadiness, EndpointReadinessOutcome, HttpResponseParts, INFERENCE_PROBE_ATTEMPTED_STATE_KEY, INFERENCE_PROBE_RETRY_INTERVAL, INFERENCE_PROBE_TIMEOUT, - INFERENCE_VERIFIED_STATE_KEY, connect_tcp_stream, download_file_streaming, + INFERENCE_VERIFIED_STATE_KEY, MultipartFile, connect_tcp_stream, download_file_streaming, download_file_streaming_with_progress, download_file_to_path, download_file_to_path_with_progress, engine_state_inference_verified, format_host_for_url, format_host_port, format_http_base_url, http_get_text, http_get_text_with_auth, - http_post_json_with_auth, managed_service_endpoint_model_ready, + http_post_json_with_auth, http_post_multipart_with_auth, managed_service_endpoint_model_ready, managed_service_endpoint_readiness, merge_json_state_file, openai_chat_completion_probe, - openai_chat_completion_status, openai_models_endpoint_has_model, parse_http_endpoint, - read_http_response_bounded, write_all_tcp_stream, + openai_chat_completion_status, openai_inference_probe, openai_inference_status, + openai_models_endpoint_has_model, openai_transcription_status, parse_http_endpoint, + read_http_response_bounded, silent_wav_16khz_mono, write_all_tcp_stream, }; pub use paths::{AppPaths, engine_plugin_dirs}; pub use proc_lifecycle::{ diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index 5bd43f3ebe..44117625e3 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -234,6 +234,13 @@ pub struct ModelRecipeRecord { } impl ModelRecipeRecord { + /// The recipe's task, parsed. [`ModelRecipeIndexDocument::validate`] rejects + /// a recipe whose task does not parse, so this only errors for a record + /// that bypassed validation. + pub fn model_task(&self) -> Result { + crate::ModelTask::parse(&self.task) + } + pub fn matches_ref(&self, model_ref: &str) -> bool { let normalized = normalize_model_ref(model_ref); normalize_model_ref(&self.canonical_model_id) == normalized @@ -298,6 +305,9 @@ impl ModelRecipeIndexDocument { for recipe in &self.recipes { require_nonempty(&recipe.canonical_model_id, "canonical_model_id")?; require_nonempty(&recipe.task, "task")?; + recipe + .model_task() + .with_context(|| format!("recipe `{}`", recipe.canonical_model_id))?; require_nonempty(&recipe.source, "source")?; require_nonempty(&recipe.revision, "revision")?; require_nonempty(&recipe.loader, "loader")?; @@ -1060,6 +1070,28 @@ mod tests { assert!(error.contains("must not be empty")); } + #[test] + fn model_recipe_index_rejects_a_task_this_build_cannot_serve() { + let mut recipe = sample_recipe_with_artifact("Qwen/Test-1B", &["test-qwen"]); + recipe.task = "embedding".to_owned(); + + let error = format!( + "{:#}", + ModelRecipeIndexDocument { + schema_version: 1, + source: Some("fixture".to_owned()), + generated_at_unix_ms: Some(123), + platforms: Vec::new(), + recipes: vec![recipe], + } + .validate() + .expect_err("an unservable task should be rejected") + ); + + assert!(error.contains("recipe `Qwen/Test-1B`")); + assert!(error.contains("unsupported model task `embedding`")); + } + #[test] fn model_recipe_index_rejects_duplicate_engine_recipes() { let mut recipe = sample_recipe_with_artifact("Qwen/Test-1B", &["test-qwen"]); diff --git a/crates/rocm-core/src/model_task.rs b/crates/rocm-core/src/model_task.rs new file mode 100644 index 0000000000..6f8227f7e6 --- /dev/null +++ b/crates/rocm-core/src/model_task.rs @@ -0,0 +1,123 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What a served model does: answer chat requests or transcribe speech. +//! +//! The task decides which OpenAI-compatible route a server exposes, how its +//! readiness is probed, and which commands may use it — `rocm chat` and the +//! dashboard chat pane need a chat model, and a speech-to-text server cannot +//! stand in for one. +//! +//! Records written before the task existed carry none, and every model served +//! then was a chat model, so a missing task deserializes as [`ModelTask::Chat`]. + +use anyhow::{Result, bail}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum ModelTask { + /// Chat completions: `POST /v1/chat/completions`. + #[default] + Chat, + /// Speech-to-text: `POST /v1/audio/transcriptions`. + Transcription, +} + +impl ModelTask { + /// The value written to catalog recipes and service records. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Chat => "chat", + Self::Transcription => "transcription", + } + } + + /// Plain-English name for user-facing output. + #[must_use] + pub const fn display_name(self) -> &'static str { + match self { + Self::Chat => "chat", + Self::Transcription => "speech-to-text", + } + } + + /// The route a client calls, relative to the server's `/v1` base URL. + #[must_use] + pub const fn api_route(self) -> &'static str { + match self { + Self::Chat => "/chat/completions", + Self::Transcription => "/audio/transcriptions", + } + } + + #[must_use] + pub const fn is_chat(self) -> bool { + matches!(self, Self::Chat) + } + + /// Parse a recipe's `task` string. + /// + /// Unknown values are an error rather than a silent `Chat`: a recipe that + /// names a task this build cannot serve must not be launched and probed as a + /// chat model. + pub fn parse(value: &str) -> Result { + match value.trim().to_ascii_lowercase().as_str() { + "chat" => Ok(Self::Chat), + "transcription" => Ok(Self::Transcription), + other => bail!("unsupported model task `{other}`; expected `chat` or `transcription`"), + } + } +} + +impl std::fmt::Display for ModelTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn missing_task_deserializes_as_chat_for_older_records() { + #[derive(Deserialize)] + struct Record { + #[serde(default)] + task: ModelTask, + } + let record: Record = serde_json::from_str("{}").expect("parse"); + assert_eq!(record.task, ModelTask::Chat); + } + + #[test] + fn task_round_trips_through_its_wire_name() { + for task in [ModelTask::Chat, ModelTask::Transcription] { + let json = serde_json::to_string(&task).expect("serialize"); + assert_eq!(json, format!("\"{}\"", task.as_str())); + assert_eq!(ModelTask::parse(task.as_str()).expect("parse"), task); + } + } + + #[test] + fn unknown_task_is_rejected_rather_than_treated_as_chat() { + let error = ModelTask::parse("embedding").expect_err("embedding is not servable"); + assert!( + error + .to_string() + .contains("unsupported model task `embedding`") + ); + } + + #[test] + fn each_task_names_its_own_route() { + assert_eq!(ModelTask::Chat.api_route(), "/chat/completions"); + assert_eq!( + ModelTask::Transcription.api_route(), + "/audio/transcriptions" + ); + } +} diff --git a/crates/rocm-core/src/net.rs b/crates/rocm-core/src/net.rs index 03b6616cb6..8d9c6712a0 100644 --- a/crates/rocm-core/src/net.rs +++ b/crates/rocm-core/src/net.rs @@ -559,6 +559,72 @@ pub fn http_post_json_with_auth( body: &serde_json::Value, endpoint_api_key: Option<&str>, timeout: Duration, +) -> Result { + let payload = serde_json::to_vec(body).context("failed to serialize HTTP JSON body")?; + http_post_bytes_with_auth( + endpoint_url, + path, + "application/json", + &payload, + endpoint_api_key, + timeout, + ) +} + +/// POST a `multipart/form-data` body of text `fields` plus one file part. +pub fn http_post_multipart_with_auth( + endpoint_url: &str, + path: &str, + fields: &[(&str, &str)], + file: MultipartFile<'_>, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + const BOUNDARY: &str = "rocm-cli-multipart-boundary-7d3f1c"; + let mut payload = Vec::new(); + for (name, value) in fields { + payload.extend_from_slice( + format!( + "--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n" + ) + .as_bytes(), + ); + } + payload.extend_from_slice( + format!( + "--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"{}\"; filename=\"{}\"\r\nContent-Type: {}\r\n\r\n", + file.field, file.filename, file.content_type + ) + .as_bytes(), + ); + payload.extend_from_slice(file.bytes); + payload.extend_from_slice(format!("\r\n--{BOUNDARY}--\r\n").as_bytes()); + http_post_bytes_with_auth( + endpoint_url, + path, + &format!("multipart/form-data; boundary={BOUNDARY}"), + &payload, + endpoint_api_key, + timeout, + ) +} + +/// The file part of a [`http_post_multipart_with_auth`] request. +#[derive(Debug, Clone, Copy)] +pub struct MultipartFile<'a> { + pub field: &'a str, + pub filename: &'a str, + pub content_type: &'a str, + pub bytes: &'a [u8], +} + +fn http_post_bytes_with_auth( + endpoint_url: &str, + path: &str, + content_type: &str, + payload: &[u8], + endpoint_api_key: Option<&str>, + timeout: Duration, ) -> Result { let deadline = Instant::now() + timeout; let (host, port) = parse_http_endpoint(endpoint_url) @@ -569,12 +635,13 @@ pub fn http_post_json_with_auth( Some(key) => format!("Authorization: Bearer {key}\r\n"), None => String::new(), }; - let payload = serde_json::to_string(body).context("failed to serialize HTTP JSON body")?; - let request = format!( - "POST {path} HTTP/1.1\r\nHost: {host_header}\r\nContent-Type: application/json\r\nAccept: application/json\r\nContent-Length: {}\r\n{auth_header}Connection: close\r\n\r\n{payload}", + let head = format!( + "POST {path} HTTP/1.1\r\nHost: {host_header}\r\nContent-Type: {content_type}\r\nAccept: application/json\r\nContent-Length: {}\r\n{auth_header}Connection: close\r\n\r\n", payload.len() ); - write_all_tcp_stream(&mut stream, request.as_bytes()) + let mut request = head.into_bytes(); + request.extend_from_slice(payload); + write_all_tcp_stream(&mut stream, &request) .with_context(|| format!("failed to write HTTP POST {path}"))?; let response = read_http_response_bounded(&mut stream, deadline) .with_context(|| format!("failed to read HTTP POST {path}"))?; @@ -659,6 +726,7 @@ pub fn merge_json_state_file(path: &Path, patch: &serde_json::Value) -> Result<( /// still leaves the throttle in place instead of freeing the next poll to spend /// another full timeout. pub fn engine_state_inference_verified( + task: crate::ModelTask, state_path: &Path, state: Option<&serde_json::Value>, endpoint_url: &str, @@ -686,7 +754,8 @@ pub fn engine_state_inference_verified( state_path, &serde_json::json!({ INFERENCE_PROBE_ATTEMPTED_STATE_KEY: now }), ); - if !openai_chat_completion_probe( + if !openai_inference_probe( + task, endpoint_url, model_ref, endpoint_api_key, @@ -771,6 +840,94 @@ pub fn openai_chat_completion_status( Ok(response.status) } +/// Ask a speech-to-text endpoint to transcribe one second of silence and +/// return the HTTP status — the transcription counterpart of +/// [`openai_chat_completion_status`]. +/// +/// Silence keeps the probe cheap and deterministic: the request exercises the +/// whole decode path (multipart upload, audio decode, model forward pass) +/// without depending on what a model makes of real speech. +pub fn openai_transcription_status( + endpoint_url: &str, + model_ref: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + let wav = silent_wav_16khz_mono(Duration::from_secs(1)); + let response = http_post_multipart_with_auth( + endpoint_url, + "/v1/audio/transcriptions", + &[("model", model_ref)], + MultipartFile { + field: "file", + filename: "silence.wav", + content_type: "audio/wav", + bytes: &wav, + }, + endpoint_api_key, + timeout, + )?; + Ok(response.status) +} + +/// The smallest real request for a model's task, as an HTTP status. +pub fn openai_inference_status( + task: crate::ModelTask, + endpoint_url: &str, + model_ref: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + match task { + crate::ModelTask::Chat => { + openai_chat_completion_status(endpoint_url, model_ref, endpoint_api_key, timeout) + } + crate::ModelTask::Transcription => { + openai_transcription_status(endpoint_url, model_ref, endpoint_api_key, timeout) + } + } +} + +/// [`openai_chat_completion_probe`]'s "answered below 500" rule, applied to the +/// request that matches the model's task. A speech-to-text server has no chat +/// route, so probing it with a chat request would read its `404` as serving. +pub fn openai_inference_probe( + task: crate::ModelTask, + endpoint_url: &str, + model_ref: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + let status = openai_inference_status(task, endpoint_url, model_ref, endpoint_api_key, timeout)?; + Ok(status < 500) +} + +/// A PCM WAV file of `duration` of silence, 16 kHz mono 16-bit — the input +/// format speech-to-text servers accept without resampling. +#[must_use] +pub fn silent_wav_16khz_mono(duration: Duration) -> Vec { + const SAMPLE_RATE: u32 = 16_000; + const BYTES_PER_SAMPLE: u16 = 2; + let samples = u32::try_from(duration.as_millis() * u128::from(SAMPLE_RATE) / 1000) + .unwrap_or(u32::MAX / u32::from(BYTES_PER_SAMPLE)); + let data_len = samples * u32::from(BYTES_PER_SAMPLE); + let mut wav = Vec::with_capacity(44 + data_len as usize); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(36 + data_len).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16_u32.to_le_bytes()); // fmt chunk size + wav.extend_from_slice(&1_u16.to_le_bytes()); // PCM + wav.extend_from_slice(&1_u16.to_le_bytes()); // mono + wav.extend_from_slice(&SAMPLE_RATE.to_le_bytes()); + wav.extend_from_slice(&(SAMPLE_RATE * u32::from(BYTES_PER_SAMPLE)).to_le_bytes()); + wav.extend_from_slice(&BYTES_PER_SAMPLE.to_le_bytes()); // block align + wav.extend_from_slice(&16_u16.to_le_bytes()); // bits per sample + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_len.to_le_bytes()); + wav.resize(44 + data_len as usize, 0); + wav +} + pub fn openai_models_endpoint_has_model( endpoint_url: &str, expected_model: Option<&str>, @@ -886,7 +1043,8 @@ pub fn managed_service_endpoint_readiness( record.canonical_model_id.as_str() }; record.inference_probe_attempted_at_unix_ms = Some(now); - if !openai_chat_completion_probe( + if !openai_inference_probe( + record.task, &record.endpoint_url, model_ref, endpoint_api_key, @@ -2244,6 +2402,92 @@ mod tests { Ok(()) } #[test] + fn a_speech_to_text_service_is_probed_with_a_transcription_not_a_chat_request() -> Result<()> { + // A speech-to-text server has no chat route; a chat probe would read its + // 404 as "serving" without ever touching the model. The probe must send + // the request the service actually exists to answer. + let (port, server) = spawn_canned_http_server(vec![ + Some(("HTTP/1.1 200 OK", MODELS_OK_BODY)), + Some(("HTTP/1.1 200 OK", r#"{"text":""}"#)), + ])?; + let mut record = probe_test_record(port); + record.task = crate::ModelTask::Transcription; + + assert_eq!( + managed_service_endpoint_readiness( + &mut record, + None, + Duration::from_secs(2), + Duration::from_secs(2) + ) + .readiness, + EndpointReadiness::Serving + ); + + let requests = server.join().expect("server thread should not panic")?; + let probe = requests.get(1).expect("an inference probe was sent"); + assert!( + probe.starts_with("POST /v1/audio/transcriptions HTTP/1.1"), + "{probe}" + ); + assert!(probe.contains("multipart/form-data; boundary=")); + assert!(probe.contains("name=\"model\"\r\n\r\nQwen/Qwen3-0.6B\r\n")); + assert!(probe.contains("filename=\"silence.wav\"")); + assert!(probe.contains("RIFF"), "the uploaded file is a WAV"); + assert!( + requests + .iter() + .all(|request| !request.contains("/v1/chat/completions")), + "no chat request may be sent to a speech-to-text service" + ); + Ok(()) + } + #[test] + fn a_speech_to_text_service_answering_5xx_is_not_ready() -> Result<()> { + let (port, server) = spawn_canned_http_server(vec![ + Some(("HTTP/1.1 200 OK", MODELS_OK_BODY)), + Some(( + "HTTP/1.1 500 Internal Server Error", + r#"{"error":"backend failed to start"}"#, + )), + ])?; + let mut record = probe_test_record(port); + record.task = crate::ModelTask::Transcription; + + assert_eq!( + managed_service_endpoint_readiness( + &mut record, + None, + Duration::from_secs(2), + Duration::from_secs(2) + ) + .readiness, + EndpointReadiness::Listing + ); + assert!(record.inference_verified_at_unix_ms.is_none()); + server.join().expect("server thread should not panic")?; + Ok(()) + } + #[test] + fn silent_wav_is_a_well_formed_16khz_mono_pcm_file() { + let wav = silent_wav_16khz_mono(Duration::from_secs(1)); + assert_eq!(wav.len(), 44 + 32_000); + assert_eq!(&wav[0..4], b"RIFF"); + assert_eq!( + u32::from_le_bytes(wav[4..8].try_into().unwrap()), + 36 + 32_000 + ); + assert_eq!(&wav[8..16], b"WAVEfmt "); + assert_eq!( + u16::from_le_bytes(wav[22..24].try_into().unwrap()), + 1, + "mono" + ); + assert_eq!(u32::from_le_bytes(wav[24..28].try_into().unwrap()), 16_000); + assert_eq!(&wav[36..40], b"data"); + assert!(wav[44..].iter().all(|byte| *byte == 0), "silence"); + } + #[test] fn a_warming_service_is_not_re_probed_on_every_poll() -> Result<()> { // Only a successful probe latches, so a model that is listed but still // loading would otherwise be re-probed by every poll — and each attempt diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index ce04590010..c944225b56 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -31,6 +31,11 @@ pub struct ManagedServiceRecord { pub engine: String, pub model_ref: String, pub canonical_model_id: String, + /// What the served model does, which decides its client route and whether + /// chat commands may use it. Records written before this field existed all + /// served chat models, which is what the default restores. + #[serde(default)] + pub task: crate::ModelTask, pub host: String, pub port: u16, pub endpoint_url: String, @@ -134,6 +139,7 @@ impl ManagedServiceRecord { engine, model_ref: model_ref.into(), canonical_model_id: canonical_model_id.into(), + task: crate::ModelTask::Chat, host, port, mode: mode.into(), diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index 00bdecc673..e7f5415e6e 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -9,6 +9,8 @@ use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; +pub use rocm_core::ModelTask; + pub const ENGINE_PROTOCOL_VERSION: &str = "0.1.0"; pub const ENGINE_RECIPE_CONTRACT_VERSION: &str = "0.1.0"; pub const ENGINE_PLUGIN_BINARY_PREFIX: &str = "rocm-engine-"; @@ -391,6 +393,10 @@ pub struct ResolveModelRequest { pub recipe_override: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub engine_recipe: Option, + /// What the model does. Omitted (and defaulted) for chat, so a request from + /// an older CLI and one for a chat model look the same to an engine. + #[serde(default, skip_serializing_if = "is_chat_task")] + pub task: ModelTask, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -407,6 +413,14 @@ pub struct LaunchRequest { pub engine_recipe: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub gpu_selection: Option, + /// What the model does; see [`ResolveModelRequest::task`]. + #[serde(default, skip_serializing_if = "is_chat_task")] + pub task: ModelTask, +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // serde's skip_serializing_if passes a reference +fn is_chat_task(task: &ModelTask) -> bool { + task.is_chat() } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -448,6 +462,22 @@ pub struct EngineCapabilities { pub tool_calling: bool, pub quantized_models: String, pub reasoning_parser: bool, + /// The model tasks this engine can serve. An engine that predates this + /// field served chat models only, which is what the default restores, so + /// `rocm serve` refuses to hand it any other kind of model. + #[serde(default = "default_engine_tasks")] + pub tasks: Vec, +} + +fn default_engine_tasks() -> Vec { + vec![ModelTask::Chat] +} + +impl EngineCapabilities { + #[must_use] + pub fn serves(&self, task: ModelTask) -> bool { + self.tasks.contains(&task) + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -607,15 +637,35 @@ impl HealthcheckResponse { /// against [`chat_supported_routes`] so the two lists agree. pub const OPENAI_CHAT_ROUTES: [&str; 3] = ["/v1/models", "/v1/chat/completions", "/v1/completions"]; -/// `supported_routes` for a chat server whose health route is `health_route`. +/// The OpenAI-compatible routes every speech-to-text server answers. +pub const OPENAI_TRANSCRIPTION_ROUTES: [&str; 2] = ["/v1/models", "/v1/audio/transcriptions"]; + +/// The OpenAI-compatible routes a server for `task` answers, besides its +/// health route. #[must_use] -pub fn chat_supported_routes(health_route: &str) -> Vec { +pub const fn task_routes(task: ModelTask) -> &'static [&'static str] { + match task { + ModelTask::Chat => &OPENAI_CHAT_ROUTES, + ModelTask::Transcription => &OPENAI_TRANSCRIPTION_ROUTES, + } +} + +/// `supported_routes` for a server for `task` whose health route is +/// `health_route`. +#[must_use] +pub fn supported_routes(health_route: &str, task: ModelTask) -> Vec { std::iter::once(health_route) - .chain(OPENAI_CHAT_ROUTES) + .chain(task_routes(task).iter().copied()) .map(str::to_owned) .collect() } +/// `supported_routes` for a chat server whose health route is `health_route`. +#[must_use] +pub fn chat_supported_routes(health_route: &str) -> Vec { + supported_routes(health_route, ModelTask::Chat) +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EndpointResponse { pub endpoint_url: String, @@ -914,6 +964,7 @@ mod tests { }], notes: vec!["signed recipe metadata".to_owned()], }), + task: ModelTask::Chat, }; let serialized = serde_json::to_string(&request).unwrap(); @@ -960,6 +1011,7 @@ mod tests { notes: Vec::new(), }), gpu_selection: None, + task: ModelTask::Chat, }; let serialized = serde_json::to_string(&request).unwrap(); @@ -1070,4 +1122,48 @@ mod tests { std::process::id() )) } + + #[test] + fn capabilities_from_an_engine_without_tasks_advertise_chat_only() { + let older = serde_json::json!({ + "cpu": false, + "rocm_gpu": true, + "openai_compatible": true, + "tool_calling": false, + "quantized_models": "none", + "reasoning_parser": false + }); + let capabilities: EngineCapabilities = serde_json::from_value(older).unwrap(); + assert_eq!(capabilities.tasks, vec![ModelTask::Chat]); + assert!(capabilities.serves(ModelTask::Chat)); + assert!(!capabilities.serves(ModelTask::Transcription)); + } + + #[test] + fn a_chat_launch_request_serializes_without_a_task_field() { + let request = LaunchRequest { + service_id: "svc".to_owned(), + env_id: None, + runtime_id: None, + model_ref: "Qwen/Test".to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: Some(DevicePolicy::GpuRequired), + endpoint_mode: None, + engine_recipe: None, + gpu_selection: None, + task: ModelTask::Chat, + }; + let chat = serde_json::to_value(&request).unwrap(); + assert!(chat.get("task").is_none(), "{chat}"); + + let transcription = serde_json::to_value(LaunchRequest { + task: ModelTask::Transcription, + ..request + }) + .unwrap(); + assert_eq!(transcription["task"], "transcription"); + let parsed: LaunchRequest = serde_json::from_value(transcription).unwrap(); + assert_eq!(parsed.task, ModelTask::Transcription); + } } diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index ac2da648a6..cb5c2d6952 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -842,7 +842,7 @@ pub(crate) fn ensure_best_llamacpp_backend( The GPU-required policy does not fall back to CPU; install a ROCm or Vulkan \ backend (e.g. `rocm engines install lemonade`) or verify the GPU driver with \ `rocm examine`.", - describe_llamacpp_backends(&backends) + describe_backend_statuses(&backends) ); }; // Record the attempted backend before the fallible install step below, so a caller @@ -863,15 +863,31 @@ pub(crate) fn ensure_best_llamacpp_backend( /// Parse `lemonade backends` table output into `(backend, status)` pairs for the /// `llamacpp` recipe only. Status is one of `installed`/`installable`/`unsupported`. fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { - const RECIPES: [&str; 8] = [ + parse_backend_statuses(output, LLAMACPP_RECIPE) +} + +/// Parse `lemonade backends` table output into `(backend, status)` pairs for one +/// recipe. Status is one of `installed`/`installable`/`unsupported`. +/// +/// Every recipe the table can list must be named here: a row whose first token is +/// not a known recipe is read as another backend of the recipe above it, so an +/// unknown recipe's rows would be attributed to its neighbour. +pub(crate) fn parse_backend_statuses(output: &str, recipe: &str) -> Vec<(String, String)> { + const RECIPES: [&str; 14] = [ + "acestep", + "embeddings", "flm", "kokoro", "llamacpp", + "moonshine", + "onnxruntime", + "openmoss", "ryzenai-llm", "sd-cpp", + "thinksound", + "trellis", "vllm", "whispercpp", - "embeddings", ]; const STATUSES: [&str; 3] = ["installed", "installable", "unsupported"]; let mut current_recipe = ""; @@ -898,7 +914,7 @@ fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { } else { (first, &tokens[1..]) }; - if current_recipe != LLAMACPP_RECIPE { + if current_recipe != recipe { continue; } if let Some(status) = rest.iter().find(|t| STATUSES.contains(t)) { @@ -911,10 +927,19 @@ fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { /// Choose the highest-priority backend Lemonade considers usable (installed or /// installable). Returns `(backend, already_installed)`. fn select_best_llamacpp_backend(backends: &[(String, String)]) -> Option<(String, bool)> { - for candidate in LLAMACPP_BACKEND_PRIORITY { + select_best_backend(backends, &LLAMACPP_BACKEND_PRIORITY) +} + +/// The first backend in `priority` that Lemonade reports as installed or +/// installable. Returns `(backend, already_installed)`. +pub(crate) fn select_best_backend( + backends: &[(String, String)], + priority: &[&str], +) -> Option<(String, bool)> { + for candidate in priority { if let Some((name, status)) = backends .iter() - .find(|(b, s)| b == candidate && (s == "installed" || s == "installable")) + .find(|(b, s)| b == *candidate && (s == "installed" || s == "installable")) { return Some((name.clone(), status == "installed")); } @@ -922,7 +947,7 @@ fn select_best_llamacpp_backend(backends: &[(String, String)]) -> Option<(String None } -fn describe_llamacpp_backends(backends: &[(String, String)]) -> String { +pub(crate) fn describe_backend_statuses(backends: &[(String, String)]) -> String { if backends.is_empty() { return "none reported".to_owned(); } @@ -933,7 +958,7 @@ fn describe_llamacpp_backends(backends: &[(String, String)]) -> String { .join(", ") } -fn run_lemonade_backends_list( +pub(crate) fn run_lemonade_backends_list( manifest: &LemonadeInstallManifest, process_env: &LemonadeProcessEnvironment, ) -> Result { diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 30c4e304f2..89621f3f4d 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -9,8 +9,8 @@ use rocm_engine_protocol::{ DetectRequest, DetectResponse, DevicePolicy, EndpointRequest, EndpointResponse, EngineCapabilities, EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, HealthcheckRequest, HealthcheckResponse, InstallRequest, InstallResponse, LaunchRequest, - LaunchResponse, LogsRequest, LogsResponse, ResolveModelRequest, ResolveModelResponse, - StopRequest, StopResponse, + LaunchResponse, LogsRequest, LogsResponse, ModelTask, ResolveModelRequest, + ResolveModelResponse, StopRequest, StopResponse, }; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -23,6 +23,7 @@ mod direct_llama; mod install; mod process; mod runtime_dir; +mod speech; mod state; #[cfg(test)] mod test_support; @@ -107,6 +108,9 @@ enum CommandKind { engine_recipe_json: Option, #[arg(long)] gpu: Option, + /// What the model does: `chat` (default) or `transcription`. + #[arg(long, default_value = "chat")] + task: String, }, } @@ -123,6 +127,7 @@ pub(crate) struct ServeHttpRequest { pub(crate) state_path: PathBuf, pub(crate) log_path: Option, pub(crate) engine_recipe: Option, + pub(crate) task: ModelTask, } pub fn run_cli() -> Result<()> { @@ -146,6 +151,7 @@ pub fn run_cli() -> Result<()> { device_policy: None, recipe_override: None, engine_recipe: None, + task: ModelTask::Chat, })?)?; } CommandKind::Launch { @@ -170,6 +176,7 @@ pub fn run_cli() -> Result<()> { endpoint_mode: Some("openai".to_owned()), engine_recipe: None, gpu_selection: crate::state::parse_gpu_selection_arg(gpu.as_deref())?, + task: ModelTask::Chat, })?)?, CommandKind::Stdio => { let envelope = read_request()?; @@ -187,6 +194,7 @@ pub fn run_cli() -> Result<()> { log_path, engine_recipe_json, gpu, + task, } => serve_http(ServeHttpRequest { service_id, model_ref, @@ -199,6 +207,7 @@ pub fn run_cli() -> Result<()> { state_path, log_path, engine_recipe: crate::state::parse_engine_recipe_json(engine_recipe_json)?, + task: ModelTask::parse(&task)?, })?, } Ok(()) @@ -221,6 +230,7 @@ pub fn builtin_serve_http( state_path: PathBuf, log_path: Option, engine_recipe: Option, + task: ModelTask, ) -> Result<()> { serve_http(ServeHttpRequest { service_id, @@ -234,6 +244,7 @@ pub fn builtin_serve_http( state_path, log_path, engine_recipe, + task, }) } @@ -306,6 +317,7 @@ fn capabilities() -> EngineCapabilities { quantized_models: "GGUF through Lemonade llama.cpp (ROCm or Vulkan GPU backend auto-selected)".to_owned(), reasoning_parser: false, + tasks: vec![ModelTask::Chat, ModelTask::Transcription], } } @@ -410,6 +422,13 @@ fn resolve_model_response(request: ResolveModelRequest) -> Result Result { state_path: state_path.clone(), log_path: Some(log_path.clone()), engine_recipe: request.engine_recipe.clone(), + task: request.task, }; let current_exe = std::env::current_exe().context("failed to discover current Lemonade engine binary")?; @@ -510,6 +530,9 @@ fn serve_http(mut request: ServeHttpRequest) -> Result<()> { process_env.gpu_indices = request.gpu_indices.clone(); let log_path = request.log_path.as_deref(); crate::state::write_running_state(&request, &runtime, std::process::id(), None, "starting")?; + if request.task == ModelTask::Transcription { + return crate::speech::serve_transcription(&request, &runtime, &process_env, log_path); + } // A canonical Hugging Face checkpoint (owner/repo:variant) cannot be served under its // exact name through Lemonade's model router (Lemonade renames registered models and // its registry has several naming quirks). Download the GGUF and run a packaged @@ -700,12 +723,18 @@ fn healthcheck_service(request: HealthcheckRequest) -> Result { + crate::process::query_loaded_model_endpoint(endpoint, &model_ref, &backend) + } + ModelTask::Transcription => { + crate::speech::query_loaded_transcription_model(endpoint, &model_ref, &backend) + } }) .transpose() .unwrap_or(None) @@ -745,7 +774,10 @@ fn endpoint_response(request: EndpointRequest) -> Result { Ok(EndpointResponse { endpoint_url, api_style: "openai".to_owned(), - supported_routes: rocm_engine_protocol::chat_supported_routes("/v1/health"), + supported_routes: rocm_engine_protocol::supported_routes( + "/v1/health", + crate::state::task_from_state(Some(&state)), + ), }) } diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs index 84f6e656aa..d67d3c334b 100644 --- a/engines/lemonade/src/process.rs +++ b/engines/lemonade/src/process.rs @@ -320,6 +320,18 @@ pub(crate) fn run_lemonade_backend_install( port: u16, backend: &str, process_env: &LemonadeProcessEnvironment, +) -> Result<()> { + run_lemonade_recipe_backend_install(manifest, host, port, LLAMACPP_RECIPE, backend, process_env) +} + +/// `lemonade backends install :` against a running server. +pub(crate) fn run_lemonade_recipe_backend_install( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + recipe: &str, + backend: &str, + process_env: &LemonadeProcessEnvironment, ) -> Result<()> { let mut command = ProcessCommand::new(&manifest.lemonade); command @@ -329,7 +341,7 @@ pub(crate) fn run_lemonade_backend_install( .arg(port.to_string()) .arg("backends") .arg("install") - .arg(format!("{LLAMACPP_RECIPE}:{backend}")) + .arg(format!("{recipe}:{backend}")) .stdin(Stdio::null()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()); @@ -403,17 +415,25 @@ pub(crate) fn run_lemonade_model_load( engine_recipe: Option<&EngineRecipeHint>, log_path: Option<&Path>, process_env: &LemonadeProcessEnvironment, +) -> Result<()> { + run_lemonade_load_command( + manifest, + &lemonade_model_load_args(host, port, model_ref, backend, engine_recipe), + log_path, + process_env, + ) +} + +/// Run a `lemonade ... load ...` invocation, sending its output to the service +/// log when there is one. +pub(crate) fn run_lemonade_load_command( + manifest: &LemonadeInstallManifest, + args: &[String], + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, ) -> Result<()> { let mut command = ProcessCommand::new(&manifest.lemonade); - command - .args(lemonade_model_load_args( - host, - port, - model_ref, - backend, - engine_recipe, - )) - .stdin(Stdio::null()); + command.args(args).stdin(Stdio::null()); if let Some(log_path) = log_path { let log = fs::OpenOptions::new() .create(true) @@ -702,7 +722,7 @@ fn lemonade_model_load_args( args } -fn query_health_json(host: &str, port: u16) -> Result { +pub(crate) fn query_health_json(host: &str, port: u16) -> Result { let endpoint = format_http_base_url(host, port); let endpoint_api_key = rocm_engine_protocol::resolve_endpoint_api_key(); let body = http_get_text_with_auth( diff --git a/engines/lemonade/src/speech.rs b/engines/lemonade/src/speech.rs new file mode 100644 index 0000000000..3152aa688a --- /dev/null +++ b/engines/lemonade/src/speech.rs @@ -0,0 +1,443 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Speech-to-text serving through Lemonade's `whispercpp` recipe. +//! +//! A transcription model always runs behind Lemonade's router (`lemond`): the +//! direct `llama-server` path the chat flow can take on Linux has no audio +//! route. The backend is chosen the same way as for chat — ROCm first, Vulkan +//! when Lemonade reports ROCm unsupported, never CPU — and is named explicitly +//! on load, because Lemonade's `auto` resolves to whatever whisper.cpp build is +//! installed, which can be the CPU one. +//! +//! What this can and cannot verify: Lemonade 11.5.1 reports a loaded model's +//! `device`, but derives it from the backend *name* — a `rocm` Whisper is +//! reported as `cpu`, and a `vulkan` one as `gpu` even where whisper.cpp found +//! no Vulkan device and ran on the CPU. So the field proves nothing either way +//! and is not consulted. What is checked is the backend Lemonade actually loaded +//! (`recipe_options.whispercpp_backend`), exactly as the chat path checks +//! `llamacpp_backend`; the GPU itself is guaranteed by `rocm serve`'s +//! usable-AMD-GPU gate, which runs before any engine is started. + +use anyhow::{Context, Result, anyhow, bail}; +use rocm_core::format_http_base_url; +use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; +use serde_json::{Value, json}; +use std::path::Path; +use std::time::Duration; + +use crate::backend_alignment::LemonadeRuntime; +use crate::process::LemonadeProcessEnvironment; +use crate::{DEFAULT_HOST, ServeHttpRequest}; + +/// Lemonade's speech-to-text recipe. +pub(crate) const WHISPERCPP_RECIPE: &str = "whispercpp"; + +/// GPU backends only, best first. `cpu` is deliberately absent: under the +/// GPU-required policy a host without a usable GPU backend fails instead of +/// transcribing on the CPU. +const WHISPERCPP_BACKEND_PRIORITY: [&str; 2] = ["rocm", "vulkan"]; + +/// How long `lemond` gets to answer its status query after it is spawned. +const LEMOND_STARTUP_TIMEOUT: Duration = Duration::from_secs(30); + +pub(crate) fn resolve_transcription_model( + canonical_model_id: String, + device_policy: DevicePolicy, + engine_recipe: Option, +) -> ResolveModelResponse { + ResolveModelResponse { + canonical_model_id, + task: rocm_engine_protocol::ModelTask::Transcription + .as_str() + .to_owned(), + source: "lemonade".to_owned(), + revision: "main".to_owned(), + loader: WHISPERCPP_RECIPE.to_owned(), + trust_remote_code: false, + chat_template_mode: "none".to_owned(), + dtype: "ggml".to_owned(), + device_policy, + estimated_memory: "under 2 GiB for Whisper Large-v3-Turbo; well under 1 GiB for the \ + smaller Whisper models" + .to_owned(), + launch_defaults: json!({ + "host": DEFAULT_HOST, + "port": rocm_core::DEFAULT_LOCAL_PORT, + "endpoint_mode": "openai" + }), + engine_recipe, + warnings: vec![ + "Lemonade runs Whisper on its ROCm whisper.cpp backend, or Vulkan when ROCm is \ + unsupported on this GPU; CPU is never used under the GPU-required policy" + .to_owned(), + ], + } +} + +/// Serve a speech-to-text model until the router exits. +pub(crate) fn serve_transcription( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + process_env: &LemonadeProcessEnvironment, + log_path: Option<&Path>, +) -> Result<()> { + let manifest = &runtime.manifest; + let mut child = + crate::process::spawn_lemond(manifest, &request.host, request.port, log_path, process_env)?; + crate::state::write_running_state( + request, + runtime, + std::process::id(), + Some(child.id()), + "running", + )?; + let prepared = prepare_transcription_model(request, runtime, process_env, log_path); + let backend = match prepared { + Ok(backend) => backend, + Err(error) => { + let _ = crate::state::terminate_pid(child.id(), true); + let _ = child.wait(); + return Err(error); + } + }; + crate::state::merge_json_state( + &request.state_path, + &json!({ + "status": "ready", + "server_pid": child.id(), + "server_start_ticks": rocm_core::process_start_ticks(child.id()), + "backend_state": "ready", + "backend_requested": backend, + "load_response": { + "status": "loaded", + "method": "lemonade-cli", + "model_name": request.model_ref, + "whispercpp_backend": backend + }, + }), + )?; + let status = child.wait().context("failed waiting for Lemonade server")?; + crate::state::mark_json_status( + &request.state_path, + if status.success() { + "stopped" + } else { + "failed" + }, + )?; + if status.success() { + Ok(()) + } else { + bail!("Lemonade server exited with status {status}") + } +} + +/// Bring the router to the point of serving: pick and install a GPU backend, +/// load the model on it, and prove one transcription succeeds. Returns the +/// backend in use. +fn prepare_transcription_model( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + process_env: &LemonadeProcessEnvironment, + log_path: Option<&Path>, +) -> Result { + let manifest = &runtime.manifest; + crate::process::wait_for_lemonade_cli_status( + manifest, + &request.host, + request.port, + LEMOND_STARTUP_TIMEOUT, + log_path, + process_env, + ) + .context("Lemonade server did not become ready")?; + let listing = crate::backend_alignment::run_lemonade_backends_list(manifest, process_env)?; + let backend = select_whispercpp_backend(&listing)?; + if backend.already_installed { + eprintln!( + "Using installed Lemonade {WHISPERCPP_RECIPE}:{} backend.", + backend.name + ); + } else { + eprintln!( + "Installing Lemonade {WHISPERCPP_RECIPE}:{} backend...", + backend.name + ); + crate::backend_alignment::install_llamacpp_backend_with_retry(|| { + crate::process::run_lemonade_recipe_backend_install( + manifest, + &request.host, + request.port, + WHISPERCPP_RECIPE, + &backend.name, + process_env, + ) + })?; + } + crate::process::run_lemonade_load_command( + manifest, + &transcription_load_args( + &request.host, + request.port, + &request.model_ref, + &backend.name, + ), + log_path, + process_env, + ) + .with_context(|| { + format!( + "failed to load {} with Lemonade {WHISPERCPP_RECIPE}:{}", + request.model_ref, backend.name + ) + })?; + let health = crate::process::query_health_json(&request.host, request.port)?; + if !health_has_transcription_model(&health, &request.model_ref, &backend.name) { + bail!( + "Lemonade load completed but the server did not report {} loaded on its \ + {WHISPERCPP_RECIPE}:{} backend", + request.model_ref, + backend.name + ); + } + let status = rocm_core::openai_transcription_status( + &format_http_base_url(&request.host, request.port), + &request.model_ref, + rocm_engine_protocol::resolve_endpoint_api_key().as_deref(), + rocm_core::INFERENCE_PROBE_TIMEOUT, + ) + .context("the post-load transcription smoke test did not complete")?; + if status != 200 { + bail!( + "{} loaded, but a test transcription returned HTTP {status}", + request.model_ref + ); + } + Ok(backend.name) +} + +#[derive(Debug, PartialEq, Eq)] +struct SelectedBackend { + name: String, + already_installed: bool, +} + +/// Pick the whisper.cpp backend from `lemonade backends` output. +fn select_whispercpp_backend(listing: &str) -> Result { + let statuses = crate::backend_alignment::parse_backend_statuses(listing, WHISPERCPP_RECIPE); + let (name, already_installed) = + crate::backend_alignment::select_best_backend(&statuses, &WHISPERCPP_BACKEND_PRIORITY) + .ok_or_else(|| { + anyhow!( + "Lemonade reports no supported GPU speech-to-text backend for this host \ + (status: {}). The GPU-required policy does not fall back to CPU; check \ + the GPU driver with `rocm examine`.", + crate::backend_alignment::describe_backend_statuses(&statuses) + ) + })?; + Ok(SelectedBackend { + name, + already_installed, + }) +} + +fn transcription_load_args(host: &str, port: u16, model_ref: &str, backend: &str) -> Vec { + vec![ + "--host".to_owned(), + host.to_owned(), + "--port".to_owned(), + port.to_string(), + "load".to_owned(), + model_ref.to_owned(), + "--whispercpp".to_owned(), + backend.to_owned(), + "--save-options".to_owned(), + ] +} + +/// Whether Lemonade reports `model_ref` loaded through `whispercpp` on exactly +/// `backend`. +fn health_has_transcription_model(health: &Value, model_ref: &str, backend: &str) -> bool { + let model_ref = model_ref.trim(); + if model_ref.is_empty() { + return false; + } + health + .get("all_models_loaded") + .and_then(Value::as_array) + .into_iter() + .flatten() + .any(|model| { + let str_field = |key: &str| model.get(key).and_then(Value::as_str); + str_field("model_name").is_some_and(|name| name.eq_ignore_ascii_case(model_ref)) + && str_field("recipe") == Some(WHISPERCPP_RECIPE) + && model + .get("recipe_options") + .and_then(|options| options.get("whispercpp_backend")) + .and_then(Value::as_str) + == Some(backend) + }) +} + +/// The healthcheck's "listed" signal for a speech-to-text service. +pub(crate) fn query_loaded_transcription_model( + endpoint_url: &str, + model_ref: &str, + backend: &str, +) -> Result { + let (host, port) = crate::state::parse_http_endpoint(endpoint_url) + .with_context(|| format!("unsupported endpoint URL `{endpoint_url}`"))?; + let health = crate::process::query_health_json(&host, port)?; + Ok(health_has_transcription_model(&health, model_ref, backend)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `lemonade backends --all` output captured from Lemonade 11.5.1 on a + /// Linux host where ROCm is unsupported, trimmed to the rows around + /// `whispercpp` plus the recipes 11.5.1 added before it. + const BACKENDS_NO_ROCM: &str = "\ +Recipe Backend Status Message/Version Action +---------------------------------------------------------------------------------------------------- +acestep cuda unsupported Unsupported GPU - + rocm unsupported Unsupported GPU - + vulkan installable Backend is supported but not installed. lemonade backends install acestep:vulkan +kokoro cpu installable Backend is supported but not installed. lemonade backends install kokoro:cpu +llamacpp cpu installable Backend is supported but not installed. lemonade backends install llamacpp:cpu + rocm unsupported Unsupported GPU - + vulkan installed b9747 - +openmoss rocm unsupported Unsupported GPU - + vulkan installable Backend is supported but not installed. lemonade backends install openmoss:vulkan +vllm rocm unsupported Unsupported GPU - +whispercpp cpu installed v1.8.4 - + metal unsupported Requires macOS - + npu unsupported Requires Windows - + rocm unsupported Unsupported GPU - + vulkan installable Backend is supported but not installed. lemonade backends install whispercpp:vulkan +"; + + #[test] + fn rocm_unsupported_selects_vulkan_even_when_the_cpu_build_is_installed() { + assert_eq!( + select_whispercpp_backend(BACKENDS_NO_ROCM).expect("vulkan is usable"), + SelectedBackend { + name: "vulkan".to_owned(), + already_installed: false, + } + ); + } + + #[test] + fn rocm_is_preferred_when_lemonade_supports_it() { + let listing = BACKENDS_NO_ROCM.replace( + " rocm unsupported Unsupported GPU -\n vulkan installable Backend is supported but not installed. lemonade backends install whispercpp:vulkan", + " rocm installed v1.8.4 -\n vulkan installable Backend is supported but not installed. lemonade backends install whispercpp:vulkan", + ); + assert_eq!( + select_whispercpp_backend(&listing).expect("rocm is usable"), + SelectedBackend { + name: "rocm".to_owned(), + already_installed: true, + } + ); + } + + #[test] + fn only_a_cpu_backend_is_refused_rather_than_used() { + let listing = "\ +Recipe Backend Status Message/Version Action +whispercpp cpu installed v1.8.4 - + rocm unsupported Unsupported GPU - + vulkan unsupported Unsupported GPU - +"; + let error = select_whispercpp_backend(listing) + .expect_err("CPU must never be selected") + .to_string(); + assert!(error.contains("does not fall back to CPU"), "{error}"); + assert!(error.contains("cpu=installed"), "{error}"); + } + + #[test] + fn rows_of_recipes_listed_before_whispercpp_are_not_mistaken_for_its_backends() { + // `acestep` and `openmoss` are 11.5.1 recipes with a usable `vulkan` + // row; were they unknown to the parser, those rows would be read as + // continuation rows of the recipe above them. + let statuses = + crate::backend_alignment::parse_backend_statuses(BACKENDS_NO_ROCM, WHISPERCPP_RECIPE); + assert_eq!( + statuses, + vec![ + ("cpu".to_owned(), "installed".to_owned()), + ("metal".to_owned(), "unsupported".to_owned()), + ("npu".to_owned(), "unsupported".to_owned()), + ("rocm".to_owned(), "unsupported".to_owned()), + ("vulkan".to_owned(), "installable".to_owned()), + ] + ); + } + + #[test] + fn load_names_the_backend_explicitly_instead_of_auto() { + assert_eq!( + transcription_load_args("127.0.0.1", 13305, "Whisper-Tiny", "rocm"), + vec![ + "--host", + "127.0.0.1", + "--port", + "13305", + "load", + "Whisper-Tiny", + "--whispercpp", + "rocm", + "--save-options", + ] + ); + } + + /// `all_models_loaded` as Lemonade 11.5.1 reported it for Whisper-Tiny + /// loaded with `--whispercpp vulkan` (captured, then trimmed). + fn loaded(backend: &str, device: &str) -> Value { + json!({ + "all_models_loaded": [{ + "model_name": "Whisper-Tiny", + "recipe": "whispercpp", + "device": device, + "type": "transcription", + "recipe_options": {"ctx_size": 4096, "whispercpp_backend": backend} + }] + }) + } + + #[test] + fn loaded_model_must_be_on_the_requested_backend() { + assert!(health_has_transcription_model( + &loaded("vulkan", "gpu"), + "Whisper-Tiny", + "vulkan" + )); + assert!( + !health_has_transcription_model(&loaded("cpu", "cpu"), "Whisper-Tiny", "vulkan"), + "a model Lemonade loaded on its CPU build is not the one we asked for" + ); + assert!(!health_has_transcription_model( + &loaded("vulkan", "gpu"), + "Whisper-Base", + "vulkan" + )); + } + + #[test] + fn the_device_label_is_not_trusted_either_way() { + // Lemonade 11.5.1 labels a ROCm whisper.cpp model `cpu`; the backend + // decides, not the label. + assert!(health_has_transcription_model( + &loaded("rocm", "cpu"), + "Whisper-Tiny", + "rocm" + )); + } +} diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index 6df1ded86d..f348b303af 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, format_http_base_url}; +use rocm_core::{AppPaths, ModelTask, format_http_base_url}; use rocm_engine_protocol::{ DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EngineDeviceAvailability, EngineRecipeHint, GpuSelection, @@ -46,6 +46,7 @@ pub(crate) fn inference_verified( return false; }; rocm_core::engine_state_inference_verified( + task_from_state(state), state_path, state, &format_http_base_url(&host, port), @@ -54,6 +55,16 @@ pub(crate) fn inference_verified( ) } +/// The task recorded in service state. State written before the task existed +/// belongs to a chat service, and an unparsable value is treated the same way +/// because the serving path only ever writes values that parse. +pub(crate) fn task_from_state(state: Option<&Value>) -> ModelTask { + state + .and_then(|value| value_string(value, "task")) + .and_then(|task| ModelTask::parse(&task).ok()) + .unwrap_or_default() +} + /// The device string reported once a model is loaded. Reflects the backend that /// actually ran (`rocm`, `vulkan`, …) and the pinned GPU ordinal when known, /// rather than a hardcoded value, so the report matches observed execution. @@ -106,6 +117,9 @@ pub(crate) fn serve_http_command_args(request: &ServeHttpRequest) -> Vec serde_json::to_string(engine_recipe).expect("engine recipe serializes"), ]); } + if !request.task.is_chat() { + args.extend(["--task".to_owned(), request.task.as_str().to_owned()]); + } args } @@ -161,6 +175,7 @@ pub(crate) fn write_running_state( "runtime_executable": runtime.manifest.lemond, "log_path": request.log_path.as_ref().map(|path| path.display().to_string()), "engine_recipe": request.engine_recipe, + "task": request.task, "started_at_unix_ms": current_unix_millis(), // Kernel start-times captured while each PID is alive. Paired with the // PIDs, they identify these exact processes across PID recycling so a @@ -737,6 +752,7 @@ mod tests { state_path: PathBuf::from("state.json"), log_path: Some(PathBuf::from("service.log")), engine_recipe: None, + task: ModelTask::Chat, }; let args = serve_http_command_args(&request); assert!(args.contains(&"--runtime-id".to_owned())); @@ -744,5 +760,30 @@ mod tests { assert!(args.contains(&"--env-id".to_owned())); assert!(args.contains(&"env".to_owned())); assert!(!args.iter().any(|arg| arg == "cpu")); + assert!( + !args.contains(&"--task".to_owned()), + "a chat launch keeps the argument list it always had" + ); + + let transcription = ServeHttpRequest { + task: ModelTask::Transcription, + ..request + }; + let args = serve_http_command_args(&transcription); + let task_at = args + .iter() + .position(|arg| arg == "--task") + .expect("a speech-to-text launch must carry its task to the child"); + assert_eq!(args[task_at + 1], "transcription"); + } + + #[test] + fn state_without_a_task_belongs_to_a_chat_service() { + assert_eq!(task_from_state(None), ModelTask::Chat); + assert_eq!(task_from_state(Some(&json!({}))), ModelTask::Chat); + assert_eq!( + task_from_state(Some(&json!({"task": "transcription"}))), + ModelTask::Transcription + ); } } diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index 6344d6317c..468c1b2b11 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -151,6 +151,7 @@ pub fn run_cli() -> Result<()> { .transpose()?, recipe_override: None, engine_recipe: None, + task: rocm_engine_protocol::ModelTask::Chat, }, )?)?, CommandKind::Launch { @@ -175,6 +176,7 @@ pub fn run_cli() -> Result<()> { endpoint_mode: Some("openai".to_owned()), engine_recipe: None, gpu_selection: crate::process::parse_gpu_selection_arg(gpu.as_deref())?, + task: rocm_engine_protocol::ModelTask::Chat, })?)?, CommandKind::Stdio => { let envelope = read_request()?; @@ -359,6 +361,7 @@ pub(crate) fn capabilities() -> EngineCapabilities { tool_calling: false, quantized_models: "vllm-supported".to_owned(), reasoning_parser: false, + tasks: vec![rocm_engine_protocol::ModelTask::Chat], } } diff --git a/engines/vllm/src/process.rs b/engines/vllm/src/process.rs index 2b86bac4e7..ddbea0b4bc 100644 --- a/engines/vllm/src/process.rs +++ b/engines/vllm/src/process.rs @@ -34,7 +34,21 @@ const STOP_SCOPE: rocm_core::KillScope = rocm_core::KillScope::Tree; /// Default time to wait for vLLM to report readiness before giving up. const DEFAULT_VLLM_READY_TIMEOUT: Duration = Duration::from_mins(5); +/// rocm-cli serves only chat models through vLLM. `rocm serve` checks the +/// engine's advertised tasks before it gets here; this is the engine's own +/// backstop for a caller that skips that check. +fn require_chat_task(task: rocm_engine_protocol::ModelTask) -> Result<()> { + if task.is_chat() { + return Ok(()); + } + bail!( + "the vLLM engine in rocm-cli serves chat models only; it cannot serve a {} model", + task.display_name() + ) +} + pub(crate) fn resolve_model_response(request: ResolveModelRequest) -> Result { + require_chat_task(request.task)?; let device_policy = normalize_vllm_device_policy(request.device_policy)?; let engine_recipe = accepted_engine_recipe(request.engine_recipe)?; Ok(ResolveModelResponse { @@ -93,6 +107,7 @@ pub(crate) fn parse_engine_recipe_json(value: Option) -> Result Result { + require_chat_task(request.task)?; let device_policy = normalize_vllm_device_policy(request.device_policy)?; let engine_recipe = accepted_engine_recipe(request.engine_recipe)?; let requested_gpu_indices = @@ -1009,6 +1024,7 @@ mod tests { device_policy: Some(DevicePolicy::GpuRequired), recipe_override: None, engine_recipe: None, + task: rocm_engine_protocol::ModelTask::Chat, })?; assert!( @@ -1660,12 +1676,50 @@ WARNING Failed to import from vllm._C with ImportError('libtorch_cuda.so: cannot device_policy: Some(DevicePolicy::GpuRequired), recipe_override: None, engine_recipe: Some(hint.clone()), + task: rocm_engine_protocol::ModelTask::Chat, })?; assert_eq!(response.engine_recipe, Some(hint)); Ok(()) } #[test] + fn resolve_and_launch_refuse_a_speech_to_text_model() { + let error = resolve_model_response(ResolveModelRequest { + model_ref: "Whisper-Tiny".to_owned(), + runtime_id: None, + device_policy: Some(DevicePolicy::GpuRequired), + recipe_override: None, + engine_recipe: None, + task: rocm_engine_protocol::ModelTask::Transcription, + }) + .expect_err("vLLM must not resolve a speech-to-text model"); + assert!( + error + .to_string() + .contains("serves chat models only; it cannot serve a speech-to-text model"), + "{error}" + ); + + let error = launch_service(LaunchRequest { + service_id: "svc".to_owned(), + env_id: None, + runtime_id: None, + model_ref: "Whisper-Tiny".to_owned(), + host: "127.0.0.1".to_owned(), + port: 0, + device_policy: Some(DevicePolicy::GpuRequired), + endpoint_mode: None, + engine_recipe: None, + gpu_selection: None, + task: rocm_engine_protocol::ModelTask::Transcription, + }) + .expect_err("vLLM must not launch a speech-to-text model"); + assert!( + error.to_string().contains("serves chat models only"), + "{error}" + ); + } + #[test] fn resolve_model_rejects_mismatched_engine_recipe() { let error = resolve_model_response(ResolveModelRequest { model_ref: "facebook/opt-125m".to_owned(), @@ -1676,6 +1730,7 @@ WARNING Failed to import from vllm._C with ImportError('libtorch_cuda.so: cannot "lemonade", ENGINE_RECIPE_CONTRACT_VERSION, )), + task: rocm_engine_protocol::ModelTask::Chat, }) .expect_err("mismatched engine recipe should fail"); @@ -1689,6 +1744,7 @@ WARNING Failed to import from vllm._C with ImportError('libtorch_cuda.so: cannot device_policy: Some(DevicePolicy::GpuRequired), recipe_override: None, engine_recipe: Some(test_engine_recipe(ENGINE_NAME, "999.0.0")), + task: rocm_engine_protocol::ModelTask::Chat, }) .expect_err("unsupported recipe contract should fail"); diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index 836a9c9073..d5fcee6333 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -255,6 +255,7 @@ fn inference_verified( model_ref: &str, ) -> bool { rocm_core::engine_state_inference_verified( + rocm_core::ModelTask::Chat, state_path, state, endpoint_url, From 0d8659114e22ca216251e596374c342e35133c34 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 20 Oct 2026 09:37:58 +0000 Subject: [PATCH 3/7] feat(serve): add Whisper recipes; keep chat paths off speech services Add Whisper-Large-v3-Turbo (alias `whisper`, `whisper-turbo`) and Whisper-Tiny as speech-to-text recipes served by Lemonade. A speech-to-text server cannot answer chat, so the commands that need a chat model now recognise one and say why instead of failing obscurely: - rocm chat --model refuses before sending any request, and the built-in assistant selection never picks a speech service. - The Console chat pane skips speech-to-text services, and its well-known-port fallback will not adopt a port one of them holds. - rocm bench load refuses a model the catalog marks as speech-to-text. - rocm services list shows the task and the transcription route for a speech-to-text server, and the assistant's services tool reports the task. Scenarios cover the chat-only engine refusal, the services list entry and its working endpoint, the chat and bench refusals, and (on the Lemonade GPU lanes) a real Whisper serve answering on the printed endpoint. Signed-off-by: Roman Inflianskas --- apps/rocm/src/chat.rs | 3 +- apps/rocm/src/dash.rs | 33 +++ apps/rocm/src/main.rs | 14 +- apps/rocm/src/providers.rs | 64 +++- apps/rocm/src/serve_cmd.rs | 57 ++++ crates/rocm-core/src/model_catalog.json | 53 ++++ crates/rocm-core/src/net.rs | 13 +- crates/rocm-dash-tui/src/app/chat.rs | 154 +++++++++- crates/rocm-engine-protocol/src/lib.rs | 2 +- tests/e2e-cucumber/features/bench.feature | 9 + tests/e2e-cucumber/features/chat.feature | 9 + .../features/model_serving.feature | 31 ++ tests/e2e-cucumber/src/mock_server.rs | 99 ++++++- tests/e2e-cucumber/tests/e2e.rs | 1 + tests/e2e-cucumber/tests/e2e/serving_steps.rs | 2 +- tests/e2e-cucumber/tests/e2e/speech_steps.rs | 275 ++++++++++++++++++ 16 files changed, 791 insertions(+), 28 deletions(-) create mode 100644 tests/e2e-cucumber/tests/e2e/speech_steps.rs diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index af5878b50f..274dd75ed8 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -2415,11 +2415,12 @@ fn render_services_tool_result_text(records: &[ManagedServiceRecord]) -> String for record in records { let _ = writeln!( output, - " - service_id={} engine={} model={} canonical_model={} status={} running_state={} endpoint={}", + " - service_id={} engine={} model={} canonical_model={} task={} status={} running_state={} endpoint={}", record.service_id, record.engine, record.model_ref, record.canonical_model_id, + record.task, record.status, managed_service_running_state(&record.status), record.endpoint_url diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 891255c1d7..3db2da80e0 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -794,6 +794,24 @@ fn ensure_bench_csv_parent(csv_path: &std::path::Path) -> Result<()> { Ok(()) } +/// `rocm bench load` drives chat completions, so a model the catalog knows to be +/// something else — a speech-to-text model — is refused up front with the +/// reason, rather than after every request in the sweep has failed. +fn ensure_bench_model_is_chat(model: &str) -> Result<()> { + let Some(recipe) = rocm_core::resolve_model_recipe(model)? else { + return Ok(()); + }; + let task = recipe.model_task()?; + if task.is_chat() { + return Ok(()); + } + anyhow::bail!( + "`{model}` is a {} model; `rocm bench load` measures chat completions and needs a chat \ + model - pass --model with the chat model the endpoint serves", + task.display_name() + ) +} + /// Entry point for `rocm bench load`. /// /// Runs a concurrency sweep against a local http:// endpoint and appends one @@ -842,6 +860,7 @@ pub fn run_bench(a: BenchLoadArgs) -> Result<()> { rocm_dash_tui::llm::pick_first_model(&body) .with_context(|| format!("no model found at {models_url} - pass --model explicitly"))? }; + ensure_bench_model_is_chat(&model)?; // Resolve the output path: default to the same `/bench/results.csv` // file the daemon tails. Rows are appended, matching the `CsvBenchTailer` @@ -1077,6 +1096,20 @@ async fn maybe_spawn_embedded_daemon( mod tests { use super::*; + #[test] + fn bench_load_refuses_a_speech_to_text_model_and_accepts_chat_models() { + let error = ensure_bench_model_is_chat("whisper") + .expect_err("a speech-to-text model cannot be load-tested with chat requests") + .to_string(); + assert!( + error.contains("`whisper` is a speech-to-text model"), + "{error}" + ); + ensure_bench_model_is_chat("qwen").expect("a chat model is accepted"); + ensure_bench_model_is_chat("some/model-not-in-the-catalog") + .expect("a model the catalog does not know is not second-guessed"); + } + fn cfg() -> RocmCliConfig { RocmCliConfig::default() } diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 76e6eccc5e..cd09289400 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -6679,7 +6679,19 @@ pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result bool { - local_service_priority(service) != usize::MAX + service.task.is_chat() && local_service_priority(service) != usize::MAX } const fn local_service_priority(service: &ManagedServiceRecord) -> usize { @@ -2253,6 +2262,51 @@ mod tests { Ok(()) } + #[test] + fn local_provider_refuses_a_speech_to_text_service_named_as_the_chat_model() -> Result<()> { + let (root, paths) = temp_app_paths("local-provider-speech-service"); + paths.ensure()?; + let server = spawn_models_only_server("Whisper-Tiny")?; + + let mut whisper = ManagedServiceRecord::new( + &paths, + "svc-whisper", + "lemonade", + "Whisper-Tiny", + "Whisper-Tiny", + "127.0.0.1", + server.port(), + "managed", + 123, + None, + None, + None, + ); + whisper.task = rocm_core::ModelTask::Transcription; + whisper.status = "ready".to_owned(); + whisper.write()?; + + let error = select_local_chat_service(&paths, Some("Whisper-Tiny")) + .expect_err("a speech-to-text service is not a chat target") + .to_string(); + assert!( + error + .contains("is served by `svc-whisper` as a speech-to-text model, not a chat model"), + "{error}" + ); + // The remediation names the built-in assistant; that serving it makes + // `rocm chat` work is what + // `local_provider_default_chat_requires_builtin_qwen_assistant` proves. + assert!(error.contains("rocm serve qwen"), "{error}"); + assert!( + select_local_chat_service(&paths, None).is_err(), + "the default assistant path never falls through to a speech service" + ); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + #[test] fn local_provider_default_chat_requires_builtin_qwen_assistant() -> Result<()> { let (root, paths) = temp_app_paths("local-provider-default-qwen"); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index a243478127..3586f5446d 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -1060,6 +1060,63 @@ mod tests { use super::*; use rocm_core::resolve_builtin_model_recipe; + #[test] + fn a_speech_to_text_model_is_refused_by_an_engine_that_only_serves_chat() { + let recipe = resolve_builtin_model_recipe("whisper").expect("whisper recipe"); + assert_eq!( + recipe.model_task().expect("valid task"), + rocm_core::ModelTask::Transcription + ); + let root = std::env::temp_dir().join(format!( + "rocm-serve-task-check-{}", + rocm_core::unix_time_millis() + )); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + + let error = ensure_engine_serves_task( + &paths, + "vllm", + rocm_core::ModelTask::Transcription, + "whisper", + Some(&recipe), + ) + .expect_err("vLLM advertises chat only") + .to_string(); + assert_eq!( + error, + "`whisper` is a speech-to-text model, which the vllm engine cannot serve; \ + serve it with `rocm serve whisper --engine lemonade`" + ); + // The named remedy passes this same check; the e2e scenario + // `@id:serve-speech-to-text-model-refused-by-chat-only-engine` runs it. + ensure_engine_serves_task( + &paths, + "lemonade", + rocm_core::ModelTask::Transcription, + "whisper", + Some(&recipe), + ) + .expect("Lemonade advertises speech-to-text"); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn unsupported_task_message_offers_no_engine_when_the_recipe_names_none_other() { + assert_eq!( + unsupported_task_message( + "lemonade", + rocm_core::ModelTask::Transcription, + "custom-asr", + None + ), + "`custom-asr` is a speech-to-text model, which the lemonade engine cannot serve" + ); + } + #[test] fn serve_engine_selection_uses_shared_recipe_when_no_override_exists() { let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index f970bbbbfb..53bc23b8a5 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -508,6 +508,59 @@ "warnings": [ "multimodal model; text chat is the default serve path" ] + }, + { + "canonical_model_id": "Whisper-Large-v3-Turbo", + "aliases": [ + "whisper", + "whisper-turbo" + ], + "task": "transcription", + "source": "recipe_index", + "revision": "main", + "loader": "whispercpp", + "trust_remote_code": false, + "dtype": "ggml", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 2, + "recommended_system_ram_gb": 8, + "quantization": "ggml f16; Lemonade whisper.cpp ROCm or Vulkan backend", + "artifact_hint": "Lemonade model id resolved and downloaded by the Lemonade engine", + "artifacts": [], + "engine_recipes": [], + "manual_alternatives": [], + "chat_template_mode": "none", + "preferred_engines": [ + "lemonade" + ], + "warnings": [ + "speech-to-text; serves POST /v1/audio/transcriptions, not chat" + ] + }, + { + "canonical_model_id": "Whisper-Tiny", + "aliases": [], + "task": "transcription", + "source": "recipe_index", + "revision": "main", + "loader": "whispercpp", + "trust_remote_code": false, + "dtype": "ggml", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 1, + "recommended_system_ram_gb": 4, + "quantization": "ggml f16; Lemonade whisper.cpp ROCm or Vulkan backend", + "artifact_hint": "Lemonade model id resolved and downloaded by the Lemonade engine", + "artifacts": [], + "engine_recipes": [], + "manual_alternatives": [], + "chat_template_mode": "none", + "preferred_engines": [ + "lemonade" + ], + "warnings": [ + "speech-to-text; smallest Whisper model, for quick checks rather than accuracy" + ] } ] } diff --git a/crates/rocm-core/src/net.rs b/crates/rocm-core/src/net.rs index 8d9c6712a0..cfb20de92c 100644 --- a/crates/rocm-core/src/net.rs +++ b/crates/rocm-core/src/net.rs @@ -888,9 +888,11 @@ pub fn openai_inference_status( } } -/// [`openai_chat_completion_probe`]'s "answered below 500" rule, applied to the -/// request that matches the model's task. A speech-to-text server has no chat -/// route, so probing it with a chat request would read its `404` as serving. +/// Whether the endpoint answers the request that matches the model's task. +/// +/// The same "answered below 500" rule as [`openai_chat_completion_probe`]. A +/// speech-to-text server has no chat route, so probing it with a chat request +/// would read its `404` as serving. pub fn openai_inference_probe( task: crate::ModelTask, endpoint_url: &str, @@ -908,8 +910,11 @@ pub fn openai_inference_probe( pub fn silent_wav_16khz_mono(duration: Duration) -> Vec { const SAMPLE_RATE: u32 = 16_000; const BYTES_PER_SAMPLE: u16 = 2; + // Keeps the RIFF size field (`36 + data_len`) inside `u32`. + const MAX_SAMPLES: u32 = (u32::MAX - 44) / BYTES_PER_SAMPLE as u32; let samples = u32::try_from(duration.as_millis() * u128::from(SAMPLE_RATE) / 1000) - .unwrap_or(u32::MAX / u32::from(BYTES_PER_SAMPLE)); + .unwrap_or(MAX_SAMPLES) + .min(MAX_SAMPLES); let data_len = samples * u32::from(BYTES_PER_SAMPLE); let mut wav = Vec::with_capacity(44 + data_len as usize); wav.extend_from_slice(b"RIFF"); diff --git a/crates/rocm-dash-tui/src/app/chat.rs b/crates/rocm-dash-tui/src/app/chat.rs index fc0224b075..339b3d50d3 100644 --- a/crates/rocm-dash-tui/src/app/chat.rs +++ b/crates/rocm-dash-tui/src/app/chat.rs @@ -170,7 +170,7 @@ pub(crate) fn pick_managed_chat_endpoint( .get("status") .and_then(serde_json::Value::as_str) .is_some_and(|s| s.eq_ignore_ascii_case("ready")); - if !ready { + if !ready || !record_serves_chat(record) { continue; } let endpoint_present = record @@ -211,18 +211,56 @@ pub(crate) fn pick_managed_chat_endpoint( Some(ManagedChatEndpoint { base_url, model }) } -/// Query the bin's read-only `services` tool and, if a ready OpenAI-compatible -/// managed service exists, return its endpoint as an [`LlmConfig`]. +/// Whether a `services` record serves chat. Records written before services +/// carried a task were all chat services, so a missing task counts as chat. +fn record_serves_chat(record: &serde_json::Value) -> bool { + record + .get("task") + .and_then(serde_json::Value::as_str) + .is_none_or(|task| task.eq_ignore_ascii_case("chat")) +} + +/// Whether a live managed service that does not serve chat — a speech-to-text +/// server, say — is bound to `base_url`'s port. /// -/// This is how the dash learns the *actual* port it launched an engine on (e.g. -/// a tool-launched vLLM on a non-default port) instead of guessing the -/// well-known defaults. The seam call is blocking, so it runs off the reactor. -/// A best-effort `/v1/models` fetch both confirms the endpoint is live and -/// supplies the served model id; on fetch failure the registry-reported model -/// is used (the service was already readiness-verified by the bin). -pub(super) async fn detect_managed_chat( +/// The well-known-port fallback finds *a* server and adopts the first model it +/// lists; on a port held by a speech-to-text server that would set the chat +/// pane up to send chat requests to Whisper. The registry knows better. +pub(crate) fn non_chat_service_owns(services_result: &serde_json::Value, base_url: &str) -> bool { + let port_of = |url: &str| reqwest::Url::parse(url).ok()?.port_or_known_default(); + let Some(port) = port_of(base_url) else { + return false; + }; + services_result + .get("structuredContent") + .and_then(|s| s.get("services")) + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter(|record| !record_serves_chat(record)) + .filter(|record| { + record + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|status| { + ["ready", "running", "starting", "recovering"] + .iter() + .any(|live| status.eq_ignore_ascii_case(live)) + }) + }) + .filter_map(|record| { + record + .get("endpoint_url") + .and_then(serde_json::Value::as_str) + }) + .any(|endpoint| port_of(endpoint) == Some(port)) +} + +/// The bin's read-only `services` tool payload. The seam call is blocking, so +/// it runs off the reactor. +async fn fetch_services( executor: Option, -) -> Option { +) -> Option { let executor = executor?; let outcome = tokio::task::spawn_blocking(move || executor.execute("services", &serde_json::json!({}))) @@ -231,7 +269,20 @@ pub(super) async fn detect_managed_chat( let crate::tool_exec::RocmToolOutcome::Result(value) = outcome else { return None; }; - let picked = pick_managed_chat_endpoint(&value)?; + Some(value) +} + +/// Query the bin's read-only `services` tool and, if a ready OpenAI-compatible +/// managed service exists, return its endpoint as an [`LlmConfig`]. +/// +/// This is how the dash learns the *actual* port it launched an engine on (e.g. +/// a tool-launched vLLM on a non-default port) instead of guessing the +/// well-known defaults. The seam call is blocking, so it runs off the reactor. +/// A best-effort `/v1/models` fetch both confirms the endpoint is live and +/// supplies the served model id; on fetch failure the registry-reported model +/// is used (the service was already readiness-verified by the bin). +async fn detect_managed_chat(services: &serde_json::Value) -> Option { + let picked = pick_managed_chat_endpoint(services)?; match fetch_first_model(&picked.base_url).await { Some(model) => Some(crate::llm::detected_llm_config(&picked.base_url, &model)), None => picked @@ -260,12 +311,21 @@ async fn detect_local_chat_with_probe( executor: Option, probe: impl Fn() -> Option<&'static str> + Send + 'static, ) -> Option { - if let Some(cfg) = detect_managed_chat(executor).await { + let services = fetch_services(executor).await; + if let Some(services) = services.as_ref() + && let Some(cfg) = detect_managed_chat(services).await + { return Some(cfg); } // TCP probe is blocking; keep it off the async reactor. let base = tokio::task::spawn_blocking(probe).await.ok().flatten()?; + if services + .as_ref() + .is_some_and(|services| non_chat_service_owns(services, base)) + { + return None; + } // Best-effort model query; fall back to the neutral default on any failure. let model = fetch_first_model(base) @@ -517,6 +577,74 @@ mod tests { ); } + #[test] + fn skips_a_ready_speech_to_text_service() { + // The newest ready Lemonade service is a speech-to-text server; the chat + // pane must keep the older chat service instead of routing chat to it. + let mut whisper = service( + "lemonade", + "http://127.0.0.1:13306/v1", + "ready", + "Whisper-Large-v3-Turbo", + 200, + ); + whisper["task"] = serde_json::json!("transcription"); + let env = services_envelope(serde_json::json!([ + service( + "lemonade", + "http://127.0.0.1:13305/v1", + "ready", + "qwen", + 100 + ), + whisper, + ])); + let picked = pick_managed_chat_endpoint(&env).expect("the chat service"); + assert_eq!(picked.base_url, "http://127.0.0.1:13305/v1"); + } + + #[test] + fn a_record_without_a_task_is_a_chat_service() { + let env = services_envelope(serde_json::json!([service( + "lemonade", + "http://127.0.0.1:13305/v1", + "ready", + "qwen", + 100 + )])); + assert!(pick_managed_chat_endpoint(&env).is_some()); + } + + #[test] + fn a_well_known_port_held_by_a_speech_to_text_service_is_not_a_chat_endpoint() { + let mut whisper = service( + "lemonade", + "http://127.0.0.1:13305/v1", + "ready", + "Whisper-Tiny", + 100, + ); + whisper["task"] = serde_json::json!("transcription"); + let env = services_envelope(serde_json::json!([whisper.clone()])); + assert!(non_chat_service_owns(&env, "http://localhost:13305/v1")); + assert!(!non_chat_service_owns(&env, "http://127.0.0.1:8000/v1")); + + whisper["status"] = serde_json::json!("stopped"); + let stopped = services_envelope(serde_json::json!([whisper])); + assert!( + !non_chat_service_owns(&stopped, "http://127.0.0.1:13305/v1"), + "a stopped speech service no longer holds the port" + ); + let chat = services_envelope(serde_json::json!([service( + "lemonade", + "http://127.0.0.1:13305/v1", + "ready", + "qwen", + 100 + )])); + assert!(!non_chat_service_owns(&chat, "http://127.0.0.1:13305/v1")); + } + #[test] fn skips_ready_record_with_empty_endpoint() { let env = services_envelope(serde_json::json!([service("vllm", "", "ready", "m", 100)])); diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index e7f5415e6e..45c4be2f05 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -419,7 +419,7 @@ pub struct LaunchRequest { } #[allow(clippy::trivially_copy_pass_by_ref)] // serde's skip_serializing_if passes a reference -fn is_chat_task(task: &ModelTask) -> bool { +const fn is_chat_task(task: &ModelTask) -> bool { task.is_chat() } diff --git a/tests/e2e-cucumber/features/bench.feature b/tests/e2e-cucumber/features/bench.feature index cbf66f5a32..ab7aadef66 100644 --- a/tests/e2e-cucumber/features/bench.feature +++ b/tests/e2e-cucumber/features/bench.feature @@ -71,3 +71,12 @@ Feature: Benchmarking a served endpoint And a model is being served on GPU When the user benchmarks the served endpoint Then the benchmark reports measured throughput + + # `rocm bench load` drives chat completions. Pointed at a speech-to-text model + # it must refuse up front with the reason, not report a sweep of failures. + @id:bench-load-refuses-a-speech-to-text-model + Scenario: bench-07 - Benchmarking a speech-to-text model is refused with the reason + Given a speech-to-text model is being served and registered with the CLI + When the user benchmarks the served endpoint + Then the benchmark is refused because the model is a speech-to-text model + And no chat request reached the speech-to-text server diff --git a/tests/e2e-cucumber/features/chat.feature b/tests/e2e-cucumber/features/chat.feature index 0c08bba008..7a54ce44d8 100644 --- a/tests/e2e-cucumber/features/chat.feature +++ b/tests/e2e-cucumber/features/chat.feature @@ -107,3 +107,12 @@ Feature: Chat and endpoint detection When the user pipes a one-shot chat prompt through the CLI Then the CLI prints the assistant's reply And the model receives the piped prompt with its whitespace intact + + # A speech-to-text server cannot answer chat. Naming it as the chat model + # must fail with the reason, before any chat request is sent to it. + @id:chat-refuses-a-speech-to-text-model + Scenario: chat-10 - Chat refuses a speech-to-text model and says why + Given a speech-to-text model is being served and registered with the CLI + When the user asks the chat CLI to use the speech-to-text model + Then chat is refused because the model is a speech-to-text model + And no chat request reached the speech-to-text server diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index df93d1fb98..81239ecd9f 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -343,3 +343,34 @@ Feature: Model serving Scenario: serve-25 - The recommended model list prints ASCII-only text When the user lists recommended models Then the model list output contains only ASCII characters + + # Speech-to-text models (Whisper) serve only through Lemonade; vLLM advertises + # chat only. The refusal is decided by the engine's own advertised tasks + # before the GPU gate, so it gates every PR (ungated) and must not be masked + # by the no-GPU message on the mock lane. The command it recommends is the + # one serve-27 runs on a GPU lane. + @id:serve-speech-to-text-model-refused-by-chat-only-engine + Scenario: serve-26 - A speech-to-text model sent to a chat-only engine is refused with a working alternative + When the user serves a speech-to-text model on an engine that only serves chat + Then serving is refused because that engine cannot serve speech-to-text models + And the user is told the command that serves it with Lemonade + + # The real thing: Lemonade's whisper.cpp backend on the GPU (ROCm, or Vulkan + # where Lemonade reports ROCm unsupported; never CPU). Runs on the Lemonade + # GPU lanes. Whisper-Tiny keeps the download under 100 MB. + @id:serve-speech-to-text-model-prints-its-transcription-endpoint @requires-gpu @requires-engine:lemonade + Scenario: serve-27 - A served speech-to-text model answers on the endpoint the CLI prints + Given a managed runtime is active + When the user serves a speech-to-text model with the engine the CLI recommended + Then the CLI prints the speech-to-text endpoint of the served model + And a recording sent to that endpoint is transcribed + + # A chat client appends /chat/completions to a listed base URL, which a + # speech-to-text server does not have; the list must name the route it does + # serve, and that route must actually answer. + @id:serve-speech-to-text-server-listed-with-its-task + Scenario: serve-28 - A running speech-to-text server is listed with its task and the endpoint it answers on + Given a speech-to-text model is being served and registered with the CLI + When the user lists running services + Then the service is listed as speech-to-text with its transcription endpoint + And a recording sent to the listed endpoint is transcribed diff --git a/tests/e2e-cucumber/src/mock_server.rs b/tests/e2e-cucumber/src/mock_server.rs index 74170ce737..e01c3508e2 100644 --- a/tests/e2e-cucumber/src/mock_server.rs +++ b/tests/e2e-cucumber/src/mock_server.rs @@ -433,6 +433,9 @@ vllm:time_per_output_token_seconds_count{{model=\"mock\"}} {tpot_count} } } +/// The fixed transcript [`MockServer::start_transcription`] answers with. +pub const MOCK_TRANSCRIPT: &str = "mock transcript of the uploaded audio"; + #[derive(Debug)] pub struct MockServer { server: ServerHandle, @@ -442,6 +445,10 @@ pub struct MockServer { chat_paths: Arc>>, /// `Some` only when started with [`MockServer::start_with_scripted_metrics`]. scripted: Option>, + /// Every URI path a speech-to-text request arrived on. Only + /// [`MockServer::start_transcription`] serves that route; every other + /// constructor leaves this empty. + transcription_paths: Arc>>, } impl MockServer { @@ -487,6 +494,7 @@ impl MockServer { last_chat_request, scripted: Some(scripted), chat_paths, + transcription_paths: Arc::new(Mutex::new(Vec::new())), } } @@ -542,6 +550,7 @@ impl MockServer { scripted: None, last_chat_request, chat_paths, + transcription_paths: Arc::new(Mutex::new(Vec::new())), } } @@ -580,9 +589,83 @@ impl MockServer { last_chat_request: Arc::new(Mutex::new(None)), chat_paths: Arc::new(Mutex::new(Vec::new())), scripted: None, + transcription_paths: Arc::new(Mutex::new(Vec::new())), } } + /// A speech-to-text server, shaped like Lemonade serving a Whisper model: + /// it lists the model, answers `POST /v1/audio/transcriptions` with a fixed + /// transcript, and has no chat route — a chat request is refused with `404`, + /// but its path is still recorded so a scenario can prove the CLI never + /// sent one. + pub async fn start_transcription(model_name: &str) -> Self { + #[derive(Clone)] + struct TranscriptionState { + model_name: String, + chat_paths: Arc>>, + transcription_paths: Arc>>, + } + async fn models(State(state): State) -> Json { + Json(json!({ + "object": "list", + "data": [{"id": state.model_name, "object": "model"}] + })) + } + async fn transcribe( + State(state): State, + uri: axum::http::Uri, + ) -> Json { + state + .transcription_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(uri.path().to_string()); + Json(json!({"text": MOCK_TRANSCRIPT})) + } + async fn refuse_chat( + State(state): State, + uri: axum::http::Uri, + ) -> StatusCode { + state + .chat_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(uri.path().to_string()); + StatusCode::NOT_FOUND + } + + let chat_paths = Arc::new(Mutex::new(Vec::new())); + let transcription_paths = Arc::new(Mutex::new(Vec::new())); + let state = TranscriptionState { + model_name: model_name.to_string(), + chat_paths: Arc::clone(&chat_paths), + transcription_paths: Arc::clone(&transcription_paths), + }; + let app = Router::new() + .route("/v1/models", get(models)) + .route("/models", get(models)) + .route("/v1/audio/transcriptions", post(transcribe)) + .route("/v1/chat/completions", post(refuse_chat)) + .route("/chat/completions", post(refuse_chat)) + .with_state(state); + Self { + server: http_server::spawn(app).await, + last_chat_request: Arc::new(Mutex::new(None)), + chat_paths, + scripted: None, + transcription_paths, + } + } + + /// Every URI path a speech-to-text request has arrived on, in order. + #[must_use] + pub fn transcription_paths(&self) -> Vec { + self.transcription_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + /// The OpenAI-compatible API root (`.../v1`) the CLI is pointed at. pub fn base_url(&self) -> String { self.server @@ -704,6 +787,13 @@ pub struct ServiceRecordOptions { /// refused before anything is stopped, which makes a failed restart /// observable without an engine. pub host: Option<&'static str>, + /// What the service does (`"transcription"` for a speech-to-text server). + /// `None` (the default) writes no `task` key at all — a record from before + /// services carried one, which the CLI reads as a chat service. An explicit + /// `null` would not do: the CLI would fail to parse the record and skip it. + pub task: Option<&'static str>, + /// The engine the record names. `None` (the default) keeps `vllm`. + pub engine: Option<&'static str>, } impl Default for ServiceRecordOptions { @@ -716,6 +806,8 @@ impl Default for ServiceRecordOptions { supervisor_start_ticks: None, runtime_id: None, host: None, + task: None, + engine: None, } } } @@ -749,9 +841,9 @@ pub fn write_service_record_with( // The CLI only extracts host:port from `endpoint_url` and appends // `/v1/models` for its readiness probe, which the mock serves. - let record = json!({ + let mut record = json!({ "service_id": "e2e-mock", - "engine": "vllm", + "engine": options.engine.unwrap_or("vllm"), "model_ref": model, "canonical_model_id": model, "host": options.host.unwrap_or("127.0.0.1"), @@ -778,6 +870,9 @@ pub fn write_service_record_with( "engine_state_path": services_dir.join("e2e-mock.state.json"), "created_at_unix_ms": 1_700_000_000_000_u64, }); + if let Some(task) = options.task { + record["task"] = json!(task); + } std::fs::write( services_dir.join("e2e-mock.json"), serde_json::to_vec_pretty(&record).expect("failed to serialize record"), diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index d26e306237..fafbfe32d7 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -41,6 +41,7 @@ mod e2e { pub mod service_stop_steps; pub mod serving_steps; pub mod skill_steps; + pub mod speech_steps; pub mod storage_steps; pub mod therock_steps; pub mod tui_driver; diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 0374dec286..70a0818aae 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -189,7 +189,7 @@ const ASSISTANT_PORT: u16 = 8001; /// (see [`wait_for_free_vram`]). Callers that only need the reset can ignore it; /// a serve that then fails to become ready reports it, because "the previous /// engine had not released the GPU yet" is otherwise invisible in the log. -async fn ensure_serve_port_free() -> String { +pub(crate) async fn ensure_serve_port_free() -> String { // Always kill any listener on the shared port — NOT just one that already // answers /v1/models. A prior scenario's vLLM that is still LOADING holds the // port and GPU memory without yet serving /v1/models; if we only checked HTTP diff --git a/tests/e2e-cucumber/tests/e2e/speech_steps.rs b/tests/e2e-cucumber/tests/e2e/speech_steps.rs new file mode 100644 index 0000000000..f36a822232 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/speech_steps.rs @@ -0,0 +1,275 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for serving speech-to-text models and for the commands that must not +//! mistake one for a chat model. + +use cucumber::{given, then, when}; +use e2e_cucumber::mock_server::{MOCK_TRANSCRIPT, MockServer, ServiceRecordOptions}; + +use crate::E2eWorld; + +/// The speech-to-text model the scenarios use: the smallest Whisper recipe, so +/// the GPU scenario's download stays small. +const SPEECH_MODEL_REF: &str = "whisper-tiny"; +const SPEECH_MODEL_ID: &str = "Whisper-Tiny"; + +fn cli_text(world: &E2eWorld) -> String { + format!( + "{}{}", + world.cli_output.as_deref().unwrap_or_default(), + world.cli_stderr.as_deref().unwrap_or_default() + ) +} + +fn record_cli(world: &mut E2eWorld, (stdout, stderr, rc): (String, String, i32)) { + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// One second of 16 kHz mono 16-bit silence, as a WAV file. +fn silent_wav() -> Vec { + let data_len: u32 = 32_000; + let mut wav = Vec::with_capacity(44 + data_len as usize); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(36 + data_len).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16_u32.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&16_000_u32.to_le_bytes()); + wav.extend_from_slice(&32_000_u32.to_le_bytes()); + wav.extend_from_slice(&2_u16.to_le_bytes()); + wav.extend_from_slice(&16_u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_len.to_le_bytes()); + wav.resize(44 + data_len as usize, 0); + wav +} + +/// POST a recording to an OpenAI-compatible transcription URL, the way an app +/// would, and return the HTTP status and parsed body. +async fn transcribe(url: &str, model: &str) -> (u16, serde_json::Value) { + const BOUNDARY: &str = "e2e-speech-boundary"; + let mut body = format!( + "--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n{model}\r\n\ + --{BOUNDARY}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"clip.wav\"\r\n\ + Content-Type: audio/wav\r\n\r\n" + ) + .into_bytes(); + body.extend_from_slice(&silent_wav()); + body.extend_from_slice(format!("\r\n--{BOUNDARY}--\r\n").as_bytes()); + let response = reqwest::Client::new() + .post(url) + .header( + "Content-Type", + format!("multipart/form-data; boundary={BOUNDARY}"), + ) + .body(body) + .timeout(std::time::Duration::from_mins(2)) + .send() + .await + .unwrap_or_else(|error| panic!("transcription request to {url} failed: {error}")); + let status = response.status().as_u16(); + let text = response.text().await.unwrap_or_default(); + let json = serde_json::from_str(&text).unwrap_or(serde_json::Value::String(text)); + (status, json) +} + +/// The URL on the output line `