diff --git a/Cargo.lock b/Cargo.lock index a46a744a0e..8077abd31f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -434,6 +434,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -504,7 +510,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9d0a013e3d3ee4edd61e779adf117944c08902d375f18630a0c5b8f95659734" dependencies = [ - "base64", + "base64 0.22.1", "bollard-stubs", "bytes", "futures-core", @@ -990,7 +996,7 @@ checksum = "96a87e18d925b19ebe0fd47ea45316abd216d81ec0879c2448c3f9a0e9da62be" dependencies = [ "Inflector", "anyhow", - "base64", + "base64 0.22.1", "clap", "console", "cucumber-codegen", @@ -1479,6 +1485,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1908,7 +1915,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -2105,6 +2112,8 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -2533,6 +2542,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "cfg_aliases 0.2.1", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -3525,9 +3546,10 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", + "futures-util", "http", "http-body", "http-body-util", @@ -3542,12 +3564,14 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.4.2", "web-sys", ] @@ -3557,7 +3581,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -3591,7 +3615,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.5.0", "web-sys", ] @@ -3603,7 +3627,7 @@ checksum = "2d9bae0f2b910cbd4ca2d53513f18aba3b6f6cb6cb0bf295e390d5396f56359d" dependencies = [ "as-any", "async-stream", - "base64", + "base64 0.22.1", "bytes", "eventsource-stream", "fastrand", @@ -3814,7 +3838,7 @@ dependencies = [ "tar", "tempfile", "ureq", - "zip", + "zip 2.4.2", ] [[package]] @@ -3846,6 +3870,7 @@ dependencies = [ "anyhow", "axum", "clap", + "reqwest 0.12.28", "rocm-core", "rocm-engine-protocol", "serde", @@ -3853,6 +3878,8 @@ dependencies = [ "sha2", "tokio", "ureq", + "utoipa", + "utoipa-swagger-ui", "windows-sys 0.61.2", ] @@ -4567,6 +4594,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4731,7 +4769,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bitflags 2.13.0", "fancy-regex", "filedescriptor", @@ -5212,6 +5250,12 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "typed-path" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ec2755506471f54baf5e4bb5499d8e510cc1caf271270e778a736bd2d7336c" + [[package]] name = "typenum" version = "1.20.1" @@ -5300,7 +5344,7 @@ version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ - "base64", + "base64 0.22.1", "flate2", "log", "once_cell", @@ -5343,6 +5387,51 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "utoipa" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8765fe27aeff71012a3f90fa474cf1df863a7d0dd81ed25de4e63fff2544994f" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d935f1c83fdf8b88f09bbe050d0cea78ea4afee6d7b0317403228c1200b2c8ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1fd2f9615838885c6ff4115fde8a2b58825dc7b4617f78d29c167377a15c92" +dependencies = [ + "base64 0.23.1", + "mime_guess", + "regex", + "serde", + "serde_json", + "utoipa", + "utoipa-swagger-ui-vendored", + "zip 9.0.5", +] + +[[package]] +name = "utoipa-swagger-ui-vendored" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552a64d6a0d251ae28bc021838800943cce60c9ec2e60ff331f54e79f696268f" + [[package]] name = "uuid" version = "1.23.3" @@ -5487,6 +5576,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-streams" version = "0.5.0" @@ -5974,7 +6076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" dependencies = [ "assert-json-diff", - "base64", + "base64 0.22.1", "deadpool", "futures", "http", @@ -6028,7 +6130,7 @@ dependencies = [ "tar", "tempfile", "toml", - "zip", + "zip 2.4.2", ] [[package]] @@ -6224,6 +6326,27 @@ dependencies = [ "zopfli", ] +[[package]] +name = "zip" +version = "9.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "nix 0.31.3", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.21" diff --git a/MANIFEST.md b/MANIFEST.md index 4fdb8bf83f..342eeafc00 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -73,6 +73,7 @@ repository. | axum-core | 0.5.6 | MIT | | backtrace | 0.3.76 | MIT OR Apache-2.0 | | base64 | 0.22.1 | MIT OR Apache-2.0 | +| base64 | 0.23.1 | MIT OR Apache-2.0 | | base64ct | 1.8.3 | Apache-2.0 OR MIT | | bit-set | 0.5.3 | MIT/Apache-2.0 | | bit-vec | 0.6.3 | MIT/Apache-2.0 | @@ -286,6 +287,7 @@ repository. | nanoid | 0.4.0 | MIT | | nix | 0.28.0 | MIT | | nix | 0.29.0 | MIT | +| nix | 0.31.3 | MIT | | nom | 7.1.3 | MIT | | nom | 8.0.0 | MIT | | nom_locate | 5.0.0 | MIT | @@ -458,6 +460,7 @@ repository. | symlink | 0.1.0 | MIT/Apache-2.0 | | syn | 1.0.109 | MIT OR Apache-2.0 | | syn | 2.0.118 | MIT OR Apache-2.0 | +| syn | 3.0.9 | MIT OR Apache-2.0 | | sync_wrapper | 1.0.2 | Apache-2.0 | | synstructure | 0.13.2 | MIT | | synthez | 0.4.0 | BlueOak-1.0.0 | @@ -512,6 +515,7 @@ repository. | tungstenite | 0.28.0 | MIT OR Apache-2.0 | | typed-builder | 0.23.2 | MIT OR Apache-2.0 | | typed-builder-macro | 0.23.2 | MIT OR Apache-2.0 | +| typed-path | 0.12.4 | MIT OR Apache-2.0 | | typenum | 1.20.1 | MIT OR Apache-2.0 | | ucd-trie | 0.1.7 | MIT OR Apache-2.0 | | uds_windows | 1.2.1 | MIT | @@ -529,6 +533,10 @@ repository. | utf-8 | 0.7.6 | MIT OR Apache-2.0 | | utf8_iter | 1.0.4 | Apache-2.0 OR MIT | | utf8parse | 0.2.2 | Apache-2.0 OR MIT | +| utoipa | 6.0.0 | MIT OR Apache-2.0 | +| utoipa-gen | 6.0.1 | MIT OR Apache-2.0 | +| utoipa-swagger-ui | 10.0.2 | MIT OR Apache-2.0 | +| utoipa-swagger-ui-vendored | 0.2.0 | MIT OR Apache-2.0 | | uuid | 1.23.3 | Apache-2.0 OR MIT | | valuable | 0.1.1 | MIT | | version_check | 0.9.5 | MIT/Apache-2.0 | @@ -544,6 +552,7 @@ repository. | wasm-bindgen-macro | 0.2.125 | MIT OR Apache-2.0 | | wasm-bindgen-macro-support | 0.2.125 | MIT OR Apache-2.0 | | wasm-bindgen-shared | 0.2.125 | MIT OR Apache-2.0 | +| wasm-streams | 0.4.2 | MIT OR Apache-2.0 | | wasm-streams | 0.5.0 | MIT OR Apache-2.0 | | web-sys | 0.3.102 | MIT OR Apache-2.0 | | web-time | 1.1.0 | MIT OR Apache-2.0 | @@ -616,6 +625,8 @@ repository. | zerovec | 0.11.6 | Unicode-3.0 | | zerovec-derive | 0.11.3 | Unicode-3.0 | | zip | 2.4.2 | MIT | +| zip | 9.0.5 | MIT | +| zlib-rs | 0.6.8 | Zlib | | zmij | 1.0.21 | MIT | | zopfli | 0.8.3 | Apache-2.0 | | zvariant | 5.12.0 | MIT | @@ -757,6 +768,14 @@ With the prek hooks installed this runs automatically whenever you commit a dependency-graph change, so the notices rarely go stale by hand; the hook skips if cargo-about is absent or not the pinned version. +Assets compiled in that are not crates — today, the Swagger UI bundle behind +the API docs of running model servers — are listed in a "Bundled web assets" +section maintained by hand at the end of `about.hbs`. `cargo-about` cannot see +them, so their licenses bypass the `about.toml` check: when a dependency bump +changes such an asset, update that section from the asset's own upstream +notices. A test in `rocmd` fails when the served Swagger UI version is not the +one the notices name. + CI verifies the committed file is current with `cargo xtask tpn --check`; a new dependency under a license not listed in `about.toml` fails the check until the license is reviewed and added (or the dependency dropped). CI pins the diff --git a/README.md b/README.md index 082a922afb..7183ebe1d1 100644 --- a/README.md +++ b/README.md @@ -798,8 +798,11 @@ services stop --yes`) or match the original flags. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table -with the status, the full inference endpoint, the API-qualified model name, and a -quick smoke test (time to first token and approximate tokens per second). Control +with the status, the full inference endpoint, a link to the server's API docs +(see API docs below), the API-qualified model name, and a +quick smoke test (time to first token and approximate tokens per second; a +speech-to-text model is checked with a test transcription before it reports +ready, and its summary names the task instead). Control returns to your shell with the server still running; manage it later with `rocm services` (below). @@ -825,6 +828,28 @@ The vLLM engine (Instinct) serves A safetensors-only id has no GGUF build, so serving it through Lemonade fails instead of silently substituting a different model. +#### Speech-to-text models + +`rocm serve whisper` serves Whisper Large-v3-Turbo through Lemonade +(`rocm serve whisper-tiny` serves the smallest Whisper model). A +speech-to-text server answers the OpenAI-compatible +`POST /v1/audio/transcriptions` route instead of chat. The deployment summary +prints that endpoint and names the task in place of the chat timing rows, and +`rocm services list` prints it beside the server's base URL: + +``` +curl http://127.0.0.1:11435/v1/audio/transcriptions \ + -F model=Whisper-Large-v3-Turbo -F file=@recording.wav +``` + +Lemonade runs Whisper on its ROCm whisper.cpp build, or its Vulkan build when it +reports ROCm unsupported for the GPU; its CPU build is never selected, and +`rocm serve` refuses to start without a usable AMD GPU. vLLM serves chat +models only, so `rocm serve whisper --engine vllm` is refused with the command +that works. `rocm chat`, the Console chat pane, and `rocm bench load` need a +chat model: they do not use a speech-to-text server. `rocm chat` and `rocm +bench load` say so when you name one; the Console chat pane skips it. + Some models (such as Llama) are gated and require Hugging Face authentication. Log in with `huggingface-cli login` or set `HF_TOKEN` in your environment before serving gated models. @@ -884,6 +909,43 @@ started while a `prune` is scanning waits for the prune. `--json` prints the service records verbatim, for scripting and for the remote orchestration below. +#### API docs + +Every running model server gets interactive API docs, served by the background +helper that `rocm serve` starts. Nothing extra needs to be run: once the docs +answer, `rocm serve` prints a `docs` link next to the endpoint, and `rocm +services list` repeats it for each running server (`--json` prints the records +alone, without it): + +``` + endpoint http://127.0.0.1:11435/v1/chat/completions + docs http://127.0.0.1:11430/services//docs/ +``` + +The page lists the routes the server answers (the model list, the engine's +health check, and chat and text completions, or for a speech-to-text server the +transcription route, whose "Try it out" takes an audio file upload), with +request examples already filled in with the served model. "Try it out" sends the request through the +docs page to the server, so it works for every engine without enabling +cross-origin access on the server; for a server started with an endpoint key, +enter the key under "Authorize" (it is kept only until the page is closed). +`http://127.0.0.1:11430/` lists every running server. vLLM also documents its +full API at its own `/docs` page, which the docs link to. + +The docs listen on loopback only, accept requests only from pages served on +this machine, and work offline. Change the port, or turn them off, in +`config.json` (`rocm config show` prints where it is): + +```json +{ "api_docs": { "enabled": true, "port": 11430 } } +``` + +The helper reads this when it starts. A `docs` link is printed only once the +docs answer for that server, so there is none when the docs are off, when +another program holds the port, or while the helper that is running was started +by a version of ROCm CLI without API docs or before the port changed; once that +helper exits, the next `rocm serve` starts a new one. + ### Remote machines (preview) Run a model on a different GPU machine and reach it from your own. Both machines diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index 5da3942696..e815b6e3ec 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -1934,6 +1934,217 @@ The following component(s) are licensed under Apache License 2.0: Apache License 2.0 ================================================================================ +The following component(s) are licensed under Apache License 2.0: + - utoipa-gen 6.0.1 + - utoipa-swagger-ui 10.0.2 + - utoipa 6.0.0 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================================ +Apache License 2.0 +================================================================================ + The following component(s) are licensed under Apache License 2.0: - rustls-platform-verifier 0.7.0 - unicode-linebreak 0.1.5 @@ -5757,6 +5968,7 @@ The following component(s) are licensed under Apache License 2.0: - atomic-waker 1.1.2 - backtrace 0.3.76 - base64 0.22.1 + - base64 0.23.1 - bitflags 1.3.2 - bitflags 2.13.0 - blocking 1.6.2 @@ -8203,6 +8415,7 @@ The following component(s) are licensed under Apache License 2.0: - serial2 0.2.37 - simdutf8 0.1.5 - syn 2.0.118 + - syn 3.0.9 - sync_wrapper 1.0.2 - thiserror-impl 1.0.69 - thiserror-impl 2.0.18 @@ -8214,6 +8427,7 @@ The following component(s) are licensed under Apache License 2.0: - unicode-ident 1.0.24 - utf8parse 0.2.2 - vte 0.15.0 + - wasm-streams 0.4.2 - wasm-streams 0.5.0 - winapi-i686-pc-windows-gnu 0.4.0 - winapi-x86_64-pc-windows-gnu 0.4.0 @@ -11839,3 +12053,130 @@ the following restrictions: 3. This notice may not be removed or altered from any source distribution. +================================================================================ +Bundled web assets +================================================================================ + +These components are not Rust crates; they are compiled into rocm-cli by a +crate listed above. Each license's full text appears above. + + - Swagger UI 5.32.6 + Licensed under Apache License 2.0. Compiled in by utoipa-swagger-ui + (through utoipa-swagger-ui-vendored) for the API docs of running model + servers. Its NOTICE file reads: + + swagger-ui + Copyright 2020-2021 SmartBear Software Inc. + + Swagger UI's bundle also contains third-party JavaScript. The notices it + retains, as distributed with Swagger UI 5.32.6 + (swagger-ui-bundle.js.LICENSE.txt in the swagger-ui-dist package; the + standalone preset's notices are a subset of these), read: + + /*! + Copyright (c) 2018 Jed Watson. + Licensed under the MIT License (MIT), see + http://jedwatson.github.io/classnames + */ + + /*! + * @description Recursive object extending + * @author Viacheslav Lotsmanov + * @license MIT + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Viacheslav Lotsmanov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + + /*! + * The buffer module from node.js, for the browser. + * + * @author Feross Aboukhadijeh + * @license MIT + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2021 Joachim Wester + * MIT license + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2022 Joachim Wester + * MIT licensed + */ + + /*! + * repeat-string + * + * Copyright (c) 2014-2015, Jon Schlinkert. + * Licensed under the MIT License. + */ + + /*! @license DOMPurify 3.4.0 | (c) Cure53 and other contributors | Released under the Apache license 2.0 and Mozilla Public License 2.0 | github.com/cure53/DOMPurify/blob/3.4.0/LICENSE */ + + /*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh */ + + /*! js-yaml 4.1.1 https://github.com/nodeca/js-yaml @license MIT */ + + /*! safe-buffer. MIT License. Feross Aboukhadijeh */ + + /** + * @license React + * react-dom.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * react.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * scheduler.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * use-sync-external-store-with-selector.production.js + * + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + diff --git a/about.hbs b/about.hbs index 0781534598..bf281fc8a8 100644 --- a/about.hbs +++ b/about.hbs @@ -30,3 +30,130 @@ The following component(s) are licensed under {{{name}}}: {{{text}}} {{/each}} + +================================================================================ +Bundled web assets +================================================================================ + +These components are not Rust crates; they are compiled into rocm-cli by a +crate listed above. Each license's full text appears above. + + - Swagger UI 5.32.6 + Licensed under Apache License 2.0. Compiled in by utoipa-swagger-ui + (through utoipa-swagger-ui-vendored) for the API docs of running model + servers. Its NOTICE file reads: + + swagger-ui + Copyright 2020-2021 SmartBear Software Inc. + + Swagger UI's bundle also contains third-party JavaScript. The notices it + retains, as distributed with Swagger UI 5.32.6 + (swagger-ui-bundle.js.LICENSE.txt in the swagger-ui-dist package; the + standalone preset's notices are a subset of these), read: + + /*! + Copyright (c) 2018 Jed Watson. + Licensed under the MIT License (MIT), see + http://jedwatson.github.io/classnames + */ + + /*! + * @description Recursive object extending + * @author Viacheslav Lotsmanov + * @license MIT + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Viacheslav Lotsmanov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + + /*! + * The buffer module from node.js, for the browser. + * + * @author Feross Aboukhadijeh + * @license MIT + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2021 Joachim Wester + * MIT license + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2022 Joachim Wester + * MIT licensed + */ + + /*! + * repeat-string + * + * Copyright (c) 2014-2015, Jon Schlinkert. + * Licensed under the MIT License. + */ + + /*! @license DOMPurify 3.4.0 | (c) Cure53 and other contributors | Released under the Apache license 2.0 and Mozilla Public License 2.0 | github.com/cure53/DOMPurify/blob/3.4.0/LICENSE */ + + /*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh */ + + /*! js-yaml 4.1.1 https://github.com/nodeca/js-yaml @license MIT */ + + /*! safe-buffer. MIT License. Feross Aboukhadijeh */ + + /** + * @license React + * react-dom.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * react.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * scheduler.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * use-sync-external-store-with-selector.production.js + * + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ diff --git a/apps/rocm/src/api_docs_link.rs b/apps/rocm/src/api_docs_link.rs new file mode 100644 index 0000000000..d15c30f4e1 --- /dev/null +++ b/apps/rocm/src/api_docs_link.rs @@ -0,0 +1,281 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The API docs link printed for a running model server. +//! +//! The background daemon serves the docs (`rocmd`'s `api_docs` module). A link +//! is only printed once the docs server has answered for that very server, so +//! the CLI never points at a page that is not there: docs turned off, a taken +//! port, or a daemon that has not come up yet all print no link. + +use rocm_core::{AppPaths, RocmCliConfig}; +use std::thread; +use std::time::{Duration, Instant}; + +/// How long one probe of the docs server may take. It is on loopback, so a live +/// server answers in milliseconds and a missing one refuses the connection at once. +const PROBE_TIMEOUT: Duration = Duration::from_millis(500); + +/// How long `rocm serve` waits for a daemon that is still starting. A cold +/// daemon has to parse its config and bind before it answers. +const STARTING_HELPER_WAIT: Duration = Duration::from_secs(5); + +const RETRY_INTERVAL: Duration = Duration::from_millis(200); + +/// How long `rocm serve` should wait for the docs: only a helper that is still +/// starting is worth waiting for. One that was already running answers at once +/// or never will — it may predate API docs, or its port may be taken — and +/// waiting on it would stall every serve. +pub(crate) const fn serve_docs_wait(helper_starting: bool) -> Duration { + if helper_starting { + STARTING_HELPER_WAIT + } else { + Duration::ZERO + } +} + +/// The docs link for `service_id`, if the docs server answers for it within +/// `wait` (one attempt when `wait` is zero). `on_wait` runs between attempts, +/// so a caller can keep its progress indicator moving. +pub(crate) fn service_docs_link( + paths: &AppPaths, + service_id: &str, + wait: Duration, + on_wait: &mut dyn FnMut(), +) -> Option { + let docs = RocmCliConfig::load(paths).ok()?.api_docs; + docs.served_port()?; + let base_url = docs.base_url(); + let deadline = Instant::now() + wait; + loop { + if docs_answer_for(&base_url, service_id) { + return Some(docs.service_docs_url(service_id)); + } + if Instant::now() + RETRY_INTERVAL > deadline { + return None; + } + on_wait(); + thread::sleep(RETRY_INTERVAL); + } +} + +/// Whether the server on the docs port answers with the description of this +/// very service. Checking what it answers, not just that it answers, keeps a +/// link from naming some other program that happens to hold the port. +fn docs_answer_for(base_url: &str, service_id: &str) -> bool { + rocm_core::http_get_text( + base_url, + &format!("/services/{service_id}/openapi.json"), + PROBE_TIMEOUT, + ) + .ok() + .and_then(|body| serde_json::from_str::(&body).ok()) + .is_some_and(|openapi| { + openapi["servers"][0]["url"].as_str() == Some(&format!("/services/{service_id}/api")) + }) +} + +/// [`service_docs_link`] for each of `service_ids`, without waiting. Asks the +/// docs server once whether it is up, so listing many servers while it is down +/// costs one refused connection rather than one per server. +pub(crate) fn service_docs_links<'a>( + paths: &AppPaths, + service_ids: impl IntoIterator, +) -> Vec<(String, String)> { + let Ok(config) = RocmCliConfig::load(paths) else { + return Vec::new(); + }; + if config.api_docs.served_port().is_none() + || rocm_core::http_get_text(&config.api_docs.base_url(), "/health", PROBE_TIMEOUT).is_err() + { + return Vec::new(); + } + service_ids + .into_iter() + .filter_map(|service_id| { + service_docs_link(paths, service_id, Duration::ZERO, &mut || {}) + .map(|link| (service_id.to_owned(), link)) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::test_paths; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + /// A stand-in docs server answering every request with `status` and `body` + /// until the returned flag is set. + fn stand_in_docs_server(status: &'static str, body: String) -> (u16, Arc) { + stand_in_docs_server_after(0, status, body) + } + + /// As [`stand_in_docs_server`], but the first `not_yet` requests are + /// answered 503, as by a docs server still coming up. + fn stand_in_docs_server_after( + not_yet: usize, + status: &'static str, + body: String, + ) -> (u16, Arc) { + let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap(); + listener.set_nonblocking(true).unwrap(); + let port = listener.local_addr().unwrap().port(); + let stop = Arc::new(AtomicBool::new(false)); + let stop_flag = Arc::clone(&stop); + thread::spawn(move || { + let mut answered = 0; + while !stop_flag.load(Ordering::Relaxed) { + let Ok((mut stream, _)) = listener.accept() else { + thread::sleep(Duration::from_millis(10)); + continue; + }; + stream.set_nonblocking(false).ok(); + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut request = [0_u8; 1024]; + let _ = stream.read(&mut request); + let (status, body) = if answered < not_yet { + ("503 Service Unavailable", "{}") + } else { + (status, body.as_str()) + }; + answered += 1; + let _ = write!( + stream, + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + } + }); + (port, stop) + } + + fn closed_port() -> u16 { + TcpListener::bind(("127.0.0.1", 0)) + .unwrap() + .local_addr() + .unwrap() + .port() + } + + fn paths_with_docs(name: &str, enabled: bool, port: u16) -> (std::path::PathBuf, AppPaths) { + let (root, paths) = test_paths(name); + paths.ensure().unwrap(); + let mut config = RocmCliConfig::default(); + config.api_docs.enabled = enabled; + config.api_docs.port = port; + config.save(&paths).unwrap(); + (root, paths) + } + + fn openapi_for(service_id: &str) -> String { + serde_json::json!({ "servers": [{ "url": format!("/services/{service_id}/api") }] }) + .to_string() + } + + #[test] + fn a_link_is_printed_when_the_docs_describe_this_server() { + let (port, stop) = stand_in_docs_server("200 OK", openapi_for("svc-1")); + let (root, paths) = paths_with_docs("docs-link-answers", true, port); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + Some(format!("http://127.0.0.1:{port}/services/svc-1/docs/")) + ); + assert_eq!( + service_docs_links(&paths, ["svc-1"]), + vec![( + "svc-1".to_owned(), + format!("http://127.0.0.1:{port}/services/svc-1/docs/") + )] + ); + stop.store(true, Ordering::Relaxed); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn no_link_when_the_docs_do_not_know_this_server() { + let (port, stop) = stand_in_docs_server("404 Not Found", "{}".to_owned()); + let (root, paths) = paths_with_docs("docs-link-404", true, port); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + None + ); + assert!(service_docs_links(&paths, ["svc-1"]).is_empty()); + stop.store(true, Ordering::Relaxed); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn no_link_when_another_program_holds_the_docs_port() { + let (port, stop) = stand_in_docs_server("200 OK", openapi_for("some-other-server")); + let (root, paths) = paths_with_docs("docs-link-squatter", true, port); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + None + ); + stop.store(true, Ordering::Relaxed); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn no_link_when_nothing_listens_on_the_docs_port() { + let (root, paths) = paths_with_docs("docs-link-closed", true, closed_port()); + let started = Instant::now(); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + None + ); + assert!(service_docs_links(&paths, ["svc-1", "svc-2"]).is_empty()); + assert!( + started.elapsed() < Duration::from_secs(2), + "a docs server that is not there must not stall the CLI" + ); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn no_link_when_docs_are_turned_off_even_if_the_port_answers() { + let (port, stop) = stand_in_docs_server("200 OK", openapi_for("svc-1")); + let (root, paths) = paths_with_docs("docs-link-disabled", false, port); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + None + ); + assert!(service_docs_links(&paths, ["svc-1"]).is_empty()); + stop.store(true, Ordering::Relaxed); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn no_link_when_the_docs_port_is_zero() { + let (root, paths) = paths_with_docs("docs-link-port-zero", true, 0); + assert_eq!( + service_docs_link(&paths, "svc-1", Duration::ZERO, &mut || {}), + None + ); + std::fs::remove_dir_all(root).ok(); + } + + #[test] + fn a_link_waits_for_a_starting_helper_but_not_for_a_running_one() { + assert_eq!(serve_docs_wait(false), Duration::ZERO); + assert!(serve_docs_wait(true) >= Duration::from_secs(1)); + + // A helper that comes up partway through the wait still gets its link, + // and the caller's progress indicator is ticked while it waits. + let (port, stop) = stand_in_docs_server_after(2, "200 OK", openapi_for("svc-1")); + let (root, paths) = paths_with_docs("docs-link-late", true, port); + let mut ticks = 0; + assert_eq!( + service_docs_link(&paths, "svc-1", serve_docs_wait(true), &mut || ticks += 1), + Some(format!("http://127.0.0.1:{port}/services/svc-1/docs/")) + ); + assert_eq!(ticks, 2, "one tick per retry"); + stop.store(true, Ordering::Relaxed); + std::fs::remove_dir_all(root).ok(); + } +} diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index af5878b50f..a9bb345dbc 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -2415,14 +2415,23 @@ fn render_services_tool_result_text(records: &[ManagedServiceRecord]) -> String for record in records { let _ = writeln!( output, - " - service_id={} engine={} model={} canonical_model={} status={} running_state={} endpoint={}", + " - service_id={} engine={} model={} canonical_model={} task={} status={} running_state={} endpoint={}{}", record.service_id, record.engine, record.model_ref, record.canonical_model_id, + record.task.display_name(), record.status, managed_service_running_state(&record.status), - record.endpoint_url + record.endpoint_url, + // A chat client appends its own route to the base URL; any other + // server is named by the route it does answer, so the assistant + // never builds a chat URL for it. + if record.task.is_chat() { + "" + } else { + record.task.api_route() + } ); } output @@ -4607,6 +4616,40 @@ endpoint: 127.0.0.1:8001 assert!(rendered.contains("status_meaning: ready/running = running")); assert!(rendered.contains("engine=vllm")); assert!(rendered.contains("running_state=running")); + assert!(rendered.contains("task=chat"), "{rendered}"); + assert!( + rendered.contains("endpoint=http://127.0.0.1:11435/v1\n"), + "{rendered}" + ); + } + + #[test] + fn services_tool_names_a_speech_to_text_server_by_its_task_and_route() { + let (_root, paths) = test_paths("services-tool-speech"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-whisper", + "lemonade", + "whisper", + "Whisper-Large-v3-Turbo", + "127.0.0.1", + 11436, + "managed", + std::process::id(), + None, + None, + Some("gpu_required".to_owned()), + ); + record.status = "ready".to_owned(); + record.task = rocm_core::ModelTask::Transcription; + + let rendered = render_services_tool_result_text(&[record]); + + assert!(rendered.contains("task=speech-to-text"), "{rendered}"); + assert!( + rendered.contains("endpoint=http://127.0.0.1:11436/v1/audio/transcriptions"), + "{rendered}" + ); } #[test] diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 891255c1d7..0b3019adff 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -794,6 +794,24 @@ fn ensure_bench_csv_parent(csv_path: &std::path::Path) -> Result<()> { Ok(()) } +/// `rocm bench load` drives chat completions, so a model the catalog knows to be +/// something else — a speech-to-text model — is refused up front with the +/// reason, rather than after every request in the sweep has failed. +fn ensure_bench_model_is_chat(model: &str) -> Result<()> { + let Some(recipe) = rocm_core::resolve_model_recipe(model)? else { + return Ok(()); + }; + let task = recipe.model_task()?; + if task.is_chat() { + return Ok(()); + } + anyhow::bail!( + "`{model}` is a {} model; `rocm bench load` measures chat completions and needs a chat \ + model - pass --model with the chat model the endpoint serves", + task.display_name() + ) +} + /// Entry point for `rocm bench load`. /// /// Runs a concurrency sweep against a local http:// endpoint and appends one @@ -842,6 +860,7 @@ pub fn run_bench(a: BenchLoadArgs) -> Result<()> { rocm_dash_tui::llm::pick_first_model(&body) .with_context(|| format!("no model found at {models_url} - pass --model explicitly"))? }; + ensure_bench_model_is_chat(&model)?; // Resolve the output path: default to the same `/bench/results.csv` // file the daemon tails. Rows are appended, matching the `CsvBenchTailer` @@ -1077,6 +1096,23 @@ async fn maybe_spawn_embedded_daemon( mod tests { use super::*; + #[test] + fn bench_load_refuses_a_speech_to_text_model_and_accepts_chat_models() { + let error = ensure_bench_model_is_chat("whisper") + .expect_err("a speech-to-text model cannot be load-tested with chat requests") + .to_string(); + assert!( + error.contains("`whisper` is a speech-to-text model"), + "{error}" + ); + // Without --model the guard sees the id the endpoint's /models lists, + // which is the canonical one rather than the alias. + assert!(ensure_bench_model_is_chat("Whisper-Tiny").is_err()); + ensure_bench_model_is_chat("qwen").expect("a chat model is accepted"); + ensure_bench_model_is_chat("some/model-not-in-the-catalog") + .expect("a model the catalog does not know is not second-guessed"); + } + fn cfg() -> RocmCliConfig { RocmCliConfig::default() } diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index f1c489ce9e..01b1509622 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4,6 +4,7 @@ #[cfg(test)] mod advised_commands; +mod api_docs_link; /// Shared archive generator and filesystem oracle for the extraction property /// tests in `comfyui` and `therock`. #[cfg(all(test, unix))] @@ -326,6 +327,8 @@ enum Command { log_path: Option, #[arg(long)] engine_recipe_json: Option, + #[arg(long, default_value = "chat")] + task: String, }, #[command(name = "__engine-stdio", hide = true)] EngineStdio { engine: String }, @@ -472,14 +475,18 @@ rocm model --verbose" /// /// Picks an engine and ROCm runtime automatically unless overridden. By default the /// server runs as a managed background service and prints a deployment summary - /// (status, endpoint, model, and smoke-test metrics); use --verbose to stream engine - /// logs in this terminal instead. When streaming, press Ctrl-D to detach and leave the + /// (status, endpoint, API docs link, model, and smoke-test metrics); use --verbose to + /// stream engine logs in this terminal instead. When streaming, press Ctrl-D to detach and leave the /// server running, or Ctrl-C to stop it. Inspect or stop servers later with /// `rocm services`. + /// + /// Speech-to-text models such as `whisper` are served through Lemonade and answer + /// on `/v1/audio/transcriptions` instead of chat; the summary prints that endpoint. #[command(after_help = "EXAMPLES:\n \ rocm serve qwen\n \ rocm serve qwen --engine vllm --port 8000\n \ -rocm serve qwen --verbose --device gpu_required")] +rocm serve qwen --verbose --device gpu_required\n \ +rocm serve whisper")] Serve { /// Model name, alias, or local model file path. model: String, @@ -624,6 +631,9 @@ rocm logs --search error timeout")] /// - collects GPU thermal/VRAM metrics every 60s for the TUI dashboard /// - listens on a local webhook port for automation events from other /// `rocm` commands + /// - serves API docs for running model servers on 127.0.0.1 (port + /// 11430 by default; `api_docs` in config.json), forwarding their + /// "Try it out" requests to the server /// /// It is normally started on demand by `rocm automations enable` and /// `rocm serve --managed`, so you rarely need to launch it yourself. Run it @@ -2233,6 +2243,7 @@ fn dispatch(cli: Cli) -> Result<()> { state_path, log_path, engine_recipe_json, + task, }) => run_builtin_engine_serve_http( &engine, service_id, @@ -2246,6 +2257,7 @@ fn dispatch(cli: Cli) -> Result<()> { state_path, log_path, parse_engine_recipe_json_arg(engine_recipe_json)?, + rocm_core::ModelTask::parse(&task)?, ), Some(Command::EngineStdio { engine }) => run_builtin_engine_stdio(&engine), Some(Command::InternalStatus) => { @@ -6658,6 +6670,13 @@ pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result Result Result> { recipe.as_ref(), &record.engine_state_path, Some(&record.log_path), + record.task, ) } @@ -7201,6 +7238,12 @@ fn restart_internal_managed_service_with( terminate: impl FnOnce(&ManagedServiceRecord) -> (Vec, bool), ) -> Result { let record = load_service_for_restart(paths, service_id, runtime_pin)?; + // Before the stop, so a restart this build cannot finish leaves the + // server running. + record + .task + .ensure_servable() + .with_context(|| format!("did not restart `{service_id}`"))?; // Preserve the endpoint key across the restart: `stop_internal_managed_service` // deletes the key file (correct on a real stop), but a restart must bring the // service back on the same public host with the same auth. Capture it first and @@ -7313,7 +7356,8 @@ fn restart_internal_managed_service_with( // Recorded before the readiness wait, so a stop issued during it finds the // new child rather than the processes already stopped. let started = record_restarted_child(paths, &stopped_record, runtime_pin, child_pid)?; - let readiness = wait_for_service_http_ready( + let readiness = wait_for_task_service_http_ready( + started.task, &started.engine, &started.host, started.port, @@ -10272,8 +10316,15 @@ fn run_builtin_engine_serve_http( state_path: PathBuf, log_path: Option, engine_recipe: Option, + task: rocm_core::ModelTask, ) -> Result<()> { let parsed_policy = parse_device_policy(Some(device_policy))?; + if engine == "vllm" && !task.is_chat() { + bail!( + "the vLLM engine in rocm-cli serves chat models only; it cannot serve a {} model", + task.display_name() + ); + } match engine { "lemonade" => rocm_engine_lemonade::builtin_serve_http( service_id, @@ -10287,6 +10338,7 @@ fn run_builtin_engine_serve_http( state_path, log_path, engine_recipe, + task, ), "vllm" => rocm_engine_vllm::builtin_serve_http( service_id, @@ -10319,6 +10371,7 @@ fn builtin_engine_serve_http_args( engine_recipe: Option<&EngineRecipeHint>, state_path: &Path, log_path: Option<&Path>, + task: rocm_core::ModelTask, ) -> Result> { let mut args = vec![ "__engine-serve-http".to_owned(), @@ -10344,6 +10397,10 @@ fn builtin_engine_serve_http_args( if let Some(log_path) = log_path { args.extend(["--log-path".to_owned(), log_path.display().to_string()]); } + // Omitted for chat so a chat launch keeps the argv it always had. + if !task.is_chat() { + args.extend(["--task".to_owned(), task.as_str().to_owned()]); + } Ok(args) } @@ -11314,6 +11371,7 @@ fn apply_app_path_env(command: &mut ProcessCommand, paths: &AppPaths) { } } +#[cfg(test)] fn wait_for_service_http_ready( engine: &str, host: &str, @@ -11321,8 +11379,30 @@ fn wait_for_service_http_ready( canonical_model_id: &str, endpoint_api_key: Option<&str>, timeout: Duration, +) -> EndpointReadiness { + wait_for_task_service_http_ready( + rocm_core::ModelTask::Chat, + engine, + host, + port, + canonical_model_id, + endpoint_api_key, + timeout, + ) +} + +/// [`wait_for_service_http_ready`] for a service of any task. +fn wait_for_task_service_http_ready( + task: rocm_core::ModelTask, + engine: &str, + host: &str, + port: u16, + canonical_model_id: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, ) -> EndpointReadiness { wait_for_service_http_ready_with_progress( + task, engine, host, port, @@ -11348,7 +11428,9 @@ fn wait_for_service_http_ready( /// maps each engine to the right listing path. `endpoint_api_key` is sent as a /// bearer token so both the listing check and the inference probe still succeed /// against a public endpoint that requires authentication. +#[allow(clippy::too_many_arguments)] fn wait_for_service_http_ready_with_progress( + task: rocm_core::ModelTask, engine: &str, host: &str, port: u16, @@ -11370,7 +11452,8 @@ fn wait_for_service_http_ready_with_progress( Duration::from_millis(750), ) .is_ok_and(|(status, body)| { - service_http_readiness_response_ready( + service_http_readiness_response_ready_for_task( + task, engine, path, status, @@ -11381,7 +11464,8 @@ fn wait_for_service_http_ready_with_progress( }); if listed { best = EndpointReadiness::Listing; - if rocm_core::openai_chat_completion_probe( + if rocm_core::openai_inference_probe( + task, &endpoint, canonical_model_id, endpoint_api_key, @@ -11485,16 +11569,47 @@ fn http_post_local_service_json( Ok((status, body.to_owned())) } +#[cfg(test)] fn service_http_readiness_response_ready( engine: &str, path: &str, status: u16, body: &str, canonical_model_id: &str, +) -> bool { + service_http_readiness_response_ready_for_task( + rocm_core::ModelTask::Chat, + engine, + path, + status, + body, + canonical_model_id, + ) +} + +/// Whether one readiness response shows the model loaded, judged for its task. +/// +/// On Lemonade a speech-to-text model is listed only by `/v1/health`, loaded +/// through `whispercpp` on a GPU backend: Lemonade's `/v1/models` lists every +/// downloaded model, loaded or not, so it says nothing about a Whisper model. +/// Any other engine that advertises speech-to-text is judged by its +/// `/v1/models`, as its chat models are; either way the transcription probe +/// that follows decides whether it actually serves. +fn service_http_readiness_response_ready_for_task( + task: rocm_core::ModelTask, + engine: &str, + path: &str, + status: u16, + body: &str, + canonical_model_id: &str, ) -> bool { if status != 200 { return false; } + if task == rocm_core::ModelTask::Transcription && engine == "lemonade" { + return path == "/v1/health" + && lemonade_health_has_transcription_model(body, canonical_model_id); + } match (engine, path) { ("lemonade", "/v1/health") => lemonade_health_ready_for_model(body, canonical_model_id), ("lemonade", "/v1/models") => model_list_ready_for_model(body, canonical_model_id, true), @@ -11521,6 +11636,28 @@ fn lemonade_health_ready_for_model(body: &str, canonical_model_id: &str) -> bool }) } +fn lemonade_health_has_transcription_model(body: &str, canonical_model_id: &str) -> bool { + let Ok(value) = serde_json::from_str::(body.trim()) else { + return false; + }; + value + .get("all_models_loaded") + .and_then(serde_json::Value::as_array) + .is_some_and(|models| { + models.iter().any(|model| { + let field = |key: &str| model.get(key).and_then(serde_json::Value::as_str); + field("model_name") + .is_some_and(|loaded| service_model_names_match(loaded, canonical_model_id)) + && field("recipe") == Some("whispercpp") + && model + .get("recipe_options") + .and_then(|options| options.get("whispercpp_backend")) + .and_then(serde_json::Value::as_str) + .is_some_and(|backend| matches!(backend, "rocm" | "vulkan")) + }) + }) +} + fn model_list_ready_for_model( body: &str, canonical_model_id: &str, @@ -13426,6 +13563,132 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn speech_to_text_readiness_waits_for_whisper_and_probes_with_a_transcription() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::{Arc, Mutex}; + + // Shaped like Lemonade serving Whisper-Tiny on `whispercpp:vulkan`, as + // captured from 11.5.1: `/v1/health` lists the model with its backend, + // and a chat request to it is refused with 400 — which the old + // chat-only wait read as serving. + const HEALTH: &str = r#"{"all_models_loaded":[{"model_name":"Whisper-Tiny","recipe":"whispercpp","device":"gpu","type":"transcription","recipe_options":{"ctx_size":4096,"whispercpp_backend":"vulkan"}}]}"#; + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + let seen = Arc::new(Mutex::new(Vec::::new())); + let server_seen = Arc::clone(&seen); + let server = thread::spawn(move || { + while let Ok((mut stream, _)) = listener.accept() { + // Drain the whole request: the probe uploads a WAV file, and + // closing on a partial read would fail its write. + stream + .set_read_timeout(Some(Duration::from_millis(200))) + .ok(); + let mut request = Vec::new(); + let mut buffer = [0_u8; 8192]; + while let Ok(read) = stream.read(&mut buffer) { + if read == 0 { + break; + } + request.extend_from_slice(&buffer[..read]); + } + let line = String::from_utf8_lossy(&request) + .lines() + .next() + .unwrap_or_default() + .to_owned(); + let (status_line, body) = if line.starts_with("GET /v1/health ") { + ("HTTP/1.1 200 OK", HEALTH) + } else if line.starts_with("POST /v1/audio/transcriptions ") { + ("HTTP/1.1 200 OK", r#"{"text":" [BLANK_AUDIO]"}"#) + } else if line.starts_with("POST /v1/chat/completions ") { + ("HTTP/1.1 400 Bad Request", r#"{"error":"not an LLM"}"#) + } else { + ("HTTP/1.1 404 Not Found", "{}") + }; + server_seen.lock().unwrap().push(line); + let _ = write!( + stream, + "{status_line}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + } + }); + + let readiness = wait_for_task_service_http_ready( + rocm_core::ModelTask::Transcription, + "lemonade", + "127.0.0.1", + port, + "Whisper-Tiny", + None, + Duration::from_secs(5), + ); + + assert_eq!(readiness, EndpointReadiness::Serving); + assert_eq!(status_for_readiness(readiness), "ready"); + let seen = seen.lock().unwrap().clone(); + assert!( + seen.iter() + .any(|line| line.starts_with("POST /v1/audio/transcriptions ")), + "readiness must be proven by a transcription: {seen:?}" + ); + assert!( + seen.iter().all(|line| !line.contains("/chat/completions")), + "no chat request may be sent to a speech-to-text server: {seen:?}" + ); + drop(server); + Ok(()) + } + + #[test] + fn a_whisper_model_on_lemonades_cpu_build_is_not_listed_as_ready() { + let health = |backend: &str| { + format!( + r#"{{"all_models_loaded":[{{"model_name":"Whisper-Tiny","recipe":"whispercpp","recipe_options":{{"whispercpp_backend":"{backend}"}}}}]}}"# + ) + }; + let ready = |body: &str| { + service_http_readiness_response_ready_for_task( + rocm_core::ModelTask::Transcription, + "lemonade", + "/v1/health", + 200, + body, + "Whisper-Tiny", + ) + }; + assert!(ready(&health("rocm"))); + assert!(ready(&health("vulkan"))); + assert!( + !ready(&health("cpu")), + "the CPU build is never a ready GPU service" + ); + assert!( + !service_http_readiness_response_ready_for_task( + rocm_core::ModelTask::Transcription, + "lemonade", + "/v1/models", + 200, + r#"{"data":[{"id":"Whisper-Tiny"}]}"#, + "Whisper-Tiny", + ), + "Lemonade lists every downloaded model, so the listing proves nothing" + ); + // Another engine that advertises speech-to-text is judged by its model + // list, as its chat models are, before the transcription probe runs. + assert!(service_http_readiness_response_ready_for_task( + rocm_core::ModelTask::Transcription, + "example-plugin", + "/v1/models", + 200, + r#"{"data":[{"id":"Whisper-Tiny"}]}"#, + "Whisper-Tiny", + )); + } + #[test] fn serve_readiness_ready_verdict_does_not_wait_for_the_peer_to_close() -> Result<()> { use std::io::{Read, Write}; @@ -17174,6 +17437,37 @@ pub(crate) mod tests { let _ = fs::remove_dir_all(root); } + #[test] + fn restart_refuses_a_task_from_a_newer_rocm_cli_before_stopping_it() { + // A task this build cannot name has no serve path here: stopping the + // server first would take it down for good. + let (root, paths) = test_paths("restart-unknown-task"); + paths.ensure().unwrap(); + let service_id = "svc-newer-task"; + let mut record = stop_test_record(&paths, service_id); + record.task = rocm_core::ModelTask::Unknown; + record.write().unwrap(); + + let mut stopped = false; + let error = restart_internal_managed_service_with(&paths, service_id, None, |_| { + stopped = true; + (Vec::new(), true) + }) + .expect_err("a task this build does not know cannot be restarted"); + let rendered = format!("{error:#}"); + let on_disk = raw_manifest(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert!( + rendered.contains("did not restart `svc-newer-task`"), + "{rendered}" + ); + assert!(rendered.contains("a newer rocm-cli"), "{rendered}"); + // The claims in that message, checked against the state they describe. + assert!(!stopped, "nothing was stopped"); + assert_eq!(on_disk.restart_count, 0, "no restart was recorded"); + assert_eq!(on_disk.task, rocm_core::ModelTask::Unknown); + } + /// A public record with a dead pid and a stored endpoint key, for the /// liveness-refresh cases below. The port is one nothing listens on, so the /// refresh's endpoint probe fails and it falls through to the pid check. @@ -18622,6 +18916,69 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn a_restarted_speech_to_text_service_is_relaunched_with_its_task() -> Result<()> { + let root = std::env::temp_dir().join(format!( + "rocm-restart-task-argv-{}", + rocm_core::unix_time_millis() + )); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-whisper", + "lemonade", + "whisper-tiny", + "Whisper-Tiny", + "127.0.0.1", + 11435, + "managed", + 0, + None, + None, + Some("gpu_required".to_owned()), + ); + record.task = rocm_core::ModelTask::Transcription; + let speech = restart_child_argv(&record)?; + record.task = rocm_core::ModelTask::Chat; + let chat = restart_child_argv(&record)?; + + assert_eq!(argv_flag_value(&speech, "--task"), Some("transcription")); + assert_eq!( + argv_flag_value(&chat, "--task"), + None, + "a chat restart keeps the argv it always had" + ); + Ok(()) + } + + #[test] + fn the_builtin_vllm_child_refuses_a_speech_to_text_model_before_starting() { + let error = run_builtin_engine_serve_http( + "vllm", + "svc".to_owned(), + "Whisper-Tiny".to_owned(), + "127.0.0.1".to_owned(), + 0, + "gpu_required", + Vec::new(), + None, + None, + PathBuf::from("state.json"), + None, + None, + rocm_core::ModelTask::Transcription, + ) + .expect_err("vLLM serves chat models only"); + assert!( + error.to_string().contains("serves chat models only"), + "{error}" + ); + } + /// The value following `flag` in an argv, or `None` when the flag is absent. fn argv_flag_value<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { let index = args.iter().position(|arg| arg == flag)?; @@ -19486,6 +19843,7 @@ pub(crate) mod tests { tool_calling: false, quantized_models: "vllm-supported".to_owned(), reasoning_parser: false, + tasks: vec![rocm_core::ModelTask::Chat], }, notes: Vec::new(), }; diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index 91834b5390..1d695a5521 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -281,8 +281,11 @@ impl ProviderAdapter for LocalProvider<'_> { fn status(&self) -> Result { let services = ready_local_services(self.paths)?; let builtin_ready = services.iter().any(local_service_is_builtin_assistant); + // Only chat models: this lists what chat can use, and a speech-to-text + // server listed here would be offered as one. let mut models = services .into_iter() + .filter(|record| record.task.is_chat()) .map(|record| record.canonical_model_id) .collect::>(); models.sort(); @@ -477,10 +480,19 @@ fn select_local_chat_service( service.canonical_model_id.eq_ignore_ascii_case(model) || service.model_ref.eq_ignore_ascii_case(model) }); - return services - .into_iter() - .next() - .context("local provider has no ready managed service for this request"); + if let Some(chat) = services.iter().position(|service| service.task.is_chat()) { + return Ok(services.swap_remove(chat)); + } + if let Some(other) = services.first() { + bail!( + "`{model}` is served by `{}` as a {} model, not a chat model; chat needs a chat \ + model: serve one (the built-in assistant is `rocm serve {BUILTIN_ASSISTANT_MODEL_ALIAS}`) \ + and run chat again without --model, or pass --model with a chat service", + other.service_id, + other.task.display_name() + ); + } + bail!("local provider has no ready managed service for this request"); } services.retain(local_service_is_builtin_assistant); services.sort_by_key(local_service_priority); @@ -493,7 +505,7 @@ fn select_local_chat_service( } const fn local_service_is_builtin_assistant(service: &ManagedServiceRecord) -> bool { - local_service_priority(service) != usize::MAX + service.task.is_chat() && local_service_priority(service) != usize::MAX } const fn local_service_priority(service: &ManagedServiceRecord) -> usize { @@ -2253,6 +2265,52 @@ mod tests { Ok(()) } + #[test] + fn local_provider_refuses_a_speech_to_text_service_named_as_the_chat_model() -> Result<()> { + let (root, paths) = temp_app_paths("local-provider-speech-service"); + paths.ensure()?; + let server = spawn_models_only_server("Whisper-Tiny")?; + + let mut whisper = ManagedServiceRecord::new( + &paths, + "svc-whisper", + "lemonade", + "Whisper-Tiny", + "Whisper-Tiny", + "127.0.0.1", + server.port(), + "managed", + 123, + None, + None, + None, + ); + whisper.task = rocm_core::ModelTask::Transcription; + whisper.status = "ready".to_owned(); + whisper.write()?; + + let error = select_local_chat_service(&paths, Some("Whisper-Tiny")) + .expect_err("a speech-to-text service is not a chat target") + .to_string(); + assert!( + error + .contains("is served by `svc-whisper` as a speech-to-text model, not a chat model"), + "{error}" + ); + // The remediation names the built-in assistant; that serving it makes + // `rocm chat` work is what + // `local_provider_default_chat_requires_builtin_qwen_assistant` proves. + assert!(error.contains("rocm serve qwen"), "{error}"); + + // Nor is it listed among the models the local chat provider offers. + let status = LocalProvider { paths: &paths }.status()?; + assert!(status.models.is_empty(), "{:?}", status.models); + assert_eq!(status.auth_status, "no_ready_local_service"); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + #[test] fn local_provider_default_chat_requires_builtin_qwen_assistant() -> Result<()> { let (root, paths) = temp_app_paths("local-provider-default-qwen"); @@ -2920,13 +2978,15 @@ mod tests { engine: "vllm".to_owned(), requested_model: "Qwen/Qwen3.5".to_owned(), api_model: "Qwen/Qwen3.5".to_owned(), - chat_endpoint: "http://127.0.0.1:8000/v1/chat/completions".to_owned(), + endpoint: "http://127.0.0.1:8000/v1/chat/completions".to_owned(), + task: rocm_core::ModelTask::Chat, service_id: "svc-ready".to_owned(), status: "ready".to_owned(), already_running: false, metrics, api_key: None, notes: Vec::new(), + docs_url: None, }); assert!(rendered.contains("time to first token n/a")); assert!(rendered.contains("throughput (approx) n/a")); @@ -3184,6 +3244,15 @@ mod tests { let request = read_http_request(&mut stream)?; if request.starts_with("GET /v1/models ") { write_models_response(&mut stream, &model_id)?; + } else if request.starts_with("POST /v1/audio/transcriptions ") { + // A speech-to-text readiness probe needs a + // transcript, as a real server gives it. + let body = r#"{"text":""}"#; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + )?; } else { write_not_found_response(&mut stream)?; } diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 275115f2ac..962b7c42a9 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -23,16 +23,16 @@ use std::fmt::Write as _; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, ModelRecipeRecord, RocmCliConfig, default_engine_for_platform, detect_host_gpu_summary, generate_service_id, preferred_serve_engine_for_host_gpu_summary, resolve_model_recipe, }; use rocm_engine_protocol::{ - DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EngineMethod, EngineRecipeEndpointHint, - EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, GpuSelection, ResolveModelRequest, - ResolveModelResponse, + DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EngineCapabilities, EngineMethod, + EngineRecipeEndpointHint, EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, + GpuSelection, ResolveModelRequest, ResolveModelResponse, }; use crate::DevicePolicyArg; @@ -577,6 +577,25 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { host_gpu_summary.as_ref(), ); let selected_engine = serve_engine.engine.clone(); + // A model outside the catalog keeps the behaviour it always had: it is + // served, and probed, as a chat model. + let task = shared_recipe + .as_ref() + .map(ModelRecipeRecord::model_task) + .transpose()? + .unwrap_or_default(); + // Before the GPU gate and any engine preparation, so a speech-to-text model + // sent to an engine that cannot serve one fails with the reason on every + // host — not with a GPU or runtime error that hides it. + if !task.is_chat() { + ensure_engine_serves_task( + &paths, + &selected_engine, + task, + &model, + shared_recipe.as_ref(), + )?; + } // Fail closed: a public bind must be authenticated, but Windows managed // Lemonade cannot receive the key (see `ensure_public_bind_engine_supported`), // so refuse rather than launch an open public server. @@ -745,6 +764,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { device_policy: Some(device_policy), recipe_override: None, engine_recipe, + task, }, )?; // Serialize GPU auto-selection with the managed-service claim: the busy-GPU @@ -779,6 +799,9 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { println!(" resolved model: {}", resolve.canonical_model_id); println!(" engine: {selected_engine}"); println!("{}", serve_engine_selection_line(&serve_engine)); + if !task.is_chat() { + println!(" task: {}", task.display_name()); + } println!(" host: {host}"); println!(" port: {port}"); if let Some(runtime_id) = resolved_selection.runtime_id.as_deref() { @@ -872,9 +895,16 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { endpoint_auth.as_deref(), launch_lock, require_api_key, + task, &mut |_elapsed| spinner.tick(), )?; - ensure_background_helper_running_quiet(summary_mode)?; + let helper_starting = ensure_background_helper_running_quiet(summary_mode)?; + let docs_link = crate::api_docs_link::service_docs_link( + &paths, + &report.service_id, + crate::api_docs_link::serve_docs_wait(helper_starting), + &mut || spinner.tick(), + ); // An equivalent service was already running, so nothing was spawned and the // freshly generated key is unused — drop it rather than leave it orphaned in @@ -898,7 +928,13 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Best-effort inference smoke test, on by default (opt out with // `--no-smoke-test`). Only meaningful for a freshly-ready server we // just launched; skipped when metrics could not be shown anyway. - let metrics = if !no_smoke_test && !report.already_running && report.status == "ready" { + // The smoke test streams a chat completion; a speech-to-text server + // was already proven by a test transcription before it reported ready. + let metrics = if task.is_chat() + && !no_smoke_test + && !report.already_running + && report.status == "ready" + { spinner.set_label("Running smoke test…"); // The local provider resolves the endpoint key from the per-service // 0600 key file by service id, so the smoke test authenticates @@ -924,13 +960,15 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { engine: selected_engine.clone(), requested_model: model, api_model: resolve.canonical_model_id, - chat_endpoint: format!("{}/chat/completions", report.endpoint_url), + endpoint: format!("{}{}", report.endpoint_url, task.api_route()), + task, service_id: report.service_id.clone(), status: report.status.clone(), already_running: report.already_running, metrics, api_key: launched_key, notes, + docs_url: docs_link, }; // Same intentional one-time key display as `print_managed_launch_plain` // in service_launch.rs; see its rationale for why the tag below is currently inert. @@ -938,7 +976,11 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { print!("{}", serve_summary::render_summary(&summary)); } else { spinner.clear(); - print_managed_launch_plain(&report, launched_key.as_deref()); + print_managed_launch_plain(&report, launched_key.as_deref(), docs_link.as_deref()); + if let Some(line) = crate::service_hints::task_endpoint_line(task, &report.endpoint_url) + { + println!("{line}"); + } } return Ok(()); } @@ -956,9 +998,57 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { endpoint_auth.as_deref(), launch_lock, require_api_key, + task, ) } +/// Refuse to hand `engine` a model whose task it does not advertise. +/// +/// The engine's own capabilities decide, so an external plugin that predates +/// tasks — and therefore reports chat only — is refused here too, rather than +/// being launched with a model it would treat as a chat model. +fn ensure_engine_serves_task( + paths: &AppPaths, + engine: &str, + task: rocm_core::ModelTask, + model: &str, + recipe: Option<&ModelRecipeRecord>, +) -> Result<()> { + let capabilities = engine_request::<_, EngineCapabilities>( + Some(paths), + engine, + EngineMethod::Capabilities, + &serde_json::json!({}), + ) + .with_context(|| format!("failed to read the capabilities of engine `{engine}`"))?; + if capabilities.serves(task) { + return Ok(()); + } + bail!("{}", unsupported_task_message(engine, task, model, recipe)) +} + +fn unsupported_task_message( + engine: &str, + task: rocm_core::ModelTask, + model: &str, + recipe: Option<&ModelRecipeRecord>, +) -> String { + let kind = task.display_name(); + let alternative = recipe.and_then(|recipe| { + recipe + .preferred_engines + .iter() + .find(|candidate| !candidate.eq_ignore_ascii_case(engine)) + }); + match alternative { + Some(alternative) => format!( + "`{model}` is a {kind} model, which the {engine} engine cannot serve; \ + serve it with `rocm serve {model} --engine {alternative}`" + ), + None => format!("`{model}` is a {kind} model, which the {engine} engine cannot serve"), + } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; @@ -966,6 +1056,63 @@ mod tests { use super::*; use rocm_core::resolve_builtin_model_recipe; + #[test] + fn a_speech_to_text_model_is_refused_by_an_engine_that_only_serves_chat() { + let recipe = resolve_builtin_model_recipe("whisper").expect("whisper recipe"); + assert_eq!( + recipe.model_task().expect("valid task"), + rocm_core::ModelTask::Transcription + ); + let root = std::env::temp_dir().join(format!( + "rocm-serve-task-check-{}", + rocm_core::unix_time_millis() + )); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + + let error = ensure_engine_serves_task( + &paths, + "vllm", + rocm_core::ModelTask::Transcription, + "whisper", + Some(&recipe), + ) + .expect_err("vLLM advertises chat only") + .to_string(); + assert_eq!( + error, + "`whisper` is a speech-to-text model, which the vllm engine cannot serve; \ + serve it with `rocm serve whisper --engine lemonade`" + ); + // The named remedy passes this same check; the e2e scenario + // `@id:serve-speech-to-text-model-refused-by-chat-only-engine` runs it. + ensure_engine_serves_task( + &paths, + "lemonade", + rocm_core::ModelTask::Transcription, + "whisper", + Some(&recipe), + ) + .expect("Lemonade advertises speech-to-text"); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn unsupported_task_message_offers_no_engine_when_the_recipe_names_none_other() { + assert_eq!( + unsupported_task_message( + "lemonade", + rocm_core::ModelTask::Transcription, + "custom-asr", + None + ), + "`custom-asr` is a speech-to-text model, which the lemonade engine cannot serve" + ); + } + #[test] fn serve_engine_selection_uses_shared_recipe_when_no_override_exists() { let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 0f1a7c968b..83ca6bf8eb 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -50,8 +50,12 @@ pub(crate) struct DeploymentSummary { /// Canonical / API-qualified model id clients pass as `"model"`. May differ /// from `requested_model` when a recipe alias resolves to a canonical id. pub api_model: String, - /// Full chat-completions endpoint, e.g. `http://127.0.0.1:1337/v1/chat/completions`. - pub chat_endpoint: String, + /// Full endpoint for the model's task, e.g. + /// `http://127.0.0.1:1337/v1/chat/completions` for a chat model or + /// `http://127.0.0.1:1337/v1/audio/transcriptions` for a speech-to-text one. + pub endpoint: String, + /// What the model does; decides which rows apply. + pub task: rocm_core::ModelTask, pub service_id: String, /// `"ready"`, `"starting"`, or an existing service's status. pub status: String, @@ -64,6 +68,8 @@ pub(crate) struct DeploymentSummary { pub api_key: Option, /// GPU/device warnings folded in from the serve plan. pub notes: Vec, + /// The server's API docs page, when the docs server answered for it. + pub docs_url: Option, } /// Render `Some(ttft)` as a human duration, or `n/a` when the probe did not @@ -121,16 +127,33 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { ] }; + // Token timings describe text generation. A speech-to-text server has none + // to report, and two permanent `n/a` rows would read as a failed + // measurement; it names its task instead, since its endpoint is not the one + // chat clients expect. + let task_rows: Vec<(&str, String)> = if summary.task.is_chat() { + vec![ + ("time to first token", format_ttft(summary.metrics.ttft)), + ("throughput (approx)", format_tps(summary.metrics.gen_tps)), + ] + } else { + vec![("task", summary.task.display_name().to_owned())] + }; + let rows: Vec<(&str, String)> = [ vec![ ("status", summary.status.clone()), ("engine", summary.engine.clone()), ], model_rows, + vec![("endpoint", summary.endpoint.clone())], + summary + .docs_url + .iter() + .map(|url| ("docs", url.clone())) + .collect(), + task_rows, vec![ - ("endpoint", summary.chat_endpoint.clone()), - ("time to first token", format_ttft(summary.metrics.ttft)), - ("throughput (approx)", format_tps(summary.metrics.gen_tps)), ("service", summary.service_id.clone()), ( "stop", @@ -262,7 +285,8 @@ mod tests { engine: "vllm".to_owned(), requested_model: "Qwen/Qwen2.5-7B-Instruct".to_owned(), api_model: "Qwen/Qwen2.5-7B-Instruct".to_owned(), - chat_endpoint: "http://127.0.0.1:1337/v1/chat/completions".to_owned(), + endpoint: "http://127.0.0.1:1337/v1/chat/completions".to_owned(), + task: rocm_core::ModelTask::Chat, service_id: "vllm-qwen-1720000000".to_owned(), status: "ready".to_owned(), already_running: false, @@ -272,9 +296,42 @@ mod tests { }, api_key: None, notes: Vec::new(), + docs_url: None, } } + // Pins only where the row goes. Whether a link is printed at all is + // `api_docs_link`'s tests; that the printed link opens the server's docs is + // the GPU serve canaries' last step (serve-05, serve-07). + #[test] + fn summary_links_the_api_docs_under_the_endpoint_when_they_answered() { + let summary = DeploymentSummary { + docs_url: Some("http://127.0.0.1:11430/services/vllm-qwen-1720000000/docs/".to_owned()), + ..base_summary() + }; + let rendered = render_summary(&summary); + let endpoint_row = rendered + .lines() + .position(|line| line.trim_start().starts_with("endpoint ")) + .expect("endpoint row"); + assert_eq!( + rendered.lines().nth(endpoint_row + 1).map(str::trim), + Some("docs http://127.0.0.1:11430/services/vllm-qwen-1720000000/docs/"), + "the docs row must follow the endpoint row" + ); + } + + #[test] + fn summary_prints_no_docs_row_when_the_docs_did_not_answer() { + let rendered = render_summary(&base_summary()); + assert!( + !rendered + .lines() + .any(|line| line.trim_start().starts_with("docs ")), + "no docs row without an answering docs server" + ); + } + #[test] fn format_ttft_uses_ms_under_a_second_and_seconds_above() { assert_eq!(format_ttft(Some(Duration::from_millis(180))), "180 ms"); @@ -310,6 +367,31 @@ mod tests { assert!(rendered.contains("rocm services stop vllm-qwen-1720000000 --yes")); } + #[test] + fn speech_to_text_summary_names_its_task_and_omits_token_timings() { + // `serve()` builds `endpoint` from the same task value it launched the + // engine with (`ModelTask::api_route`), and skips the chat smoke test + // for this task; the e2e scenario + // `@id:serve-speech-to-text-model-prints-its-transcription-endpoint` + // proves the printed route is the one the running server answers. + let summary = DeploymentSummary { + engine: "lemonade".to_owned(), + requested_model: "whisper".to_owned(), + api_model: "Whisper-Large-v3-Turbo".to_owned(), + endpoint: "http://127.0.0.1:1337/v1/audio/transcriptions".to_owned(), + task: rocm_core::ModelTask::Transcription, + metrics: SmokeMetrics::default(), + ..base_summary() + }; + let rendered = render_summary(&summary); + assert!(rendered.contains("http://127.0.0.1:1337/v1/audio/transcriptions")); + assert!(rendered.contains("task"), "{rendered}"); + assert!(rendered.contains("speech-to-text"), "{rendered}"); + assert!(!rendered.contains("time to first token"), "{rendered}"); + assert!(!rendered.contains("throughput"), "{rendered}"); + assert!(!rendered.contains("chat/completions"), "{rendered}"); + } + #[test] fn summary_structure_is_identical_across_engines() { // The same fields render in the same order regardless of engine; only the @@ -333,7 +415,7 @@ mod tests { lemonade.engine = "lemonade".to_owned(); lemonade.requested_model = "Qwen3-0.6B-GGUF".to_owned(); lemonade.api_model = "Qwen3-0.6B-GGUF".to_owned(); - lemonade.chat_endpoint = "http://127.0.0.1:8000/v1/chat/completions".to_owned(); + lemonade.endpoint = "http://127.0.0.1:8000/v1/chat/completions".to_owned(); lemonade.metrics = SmokeMetrics::default(); // engine reported no usage assert_eq!( diff --git a/apps/rocm/src/service_hints.rs b/apps/rocm/src/service_hints.rs index 6bec1bbe65..913a63e9aa 100644 --- a/apps/rocm/src/service_hints.rs +++ b/apps/rocm/src/service_hints.rs @@ -33,3 +33,38 @@ pub(crate) fn service_stop_hint(service_id: &str) -> String { pub(crate) fn remote_service_stop_hint(remote_cli: &str, service_id: &str) -> String { format!("{remote_cli} services stop {service_id} --yes") } + +/// The ` endpoint: ` line printed beside a server's `endpoint` +/// line when the server does not serve chat: chat clients append +/// `/chat/completions` to the base URL, which such a server does not have, so +/// this names the route it does answer. `None` for chat, and for a task this +/// build cannot name a route for. Every surface that prints it calls this. +pub(crate) fn task_endpoint_line(task: rocm_core::ModelTask, endpoint_url: &str) -> Option { + (!task.is_chat() && !task.api_route().is_empty()).then(|| { + format!( + " {} endpoint: {endpoint_url}{}", + task.display_name(), + task.api_route() + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use rocm_core::ModelTask; + + #[test] + fn only_a_server_that_does_not_serve_chat_gets_a_task_endpoint_line() { + assert_eq!( + task_endpoint_line(ModelTask::Transcription, "http://127.0.0.1:11435/v1").as_deref(), + Some(" speech-to-text endpoint: http://127.0.0.1:11435/v1/audio/transcriptions") + ); + assert_eq!(task_endpoint_line(ModelTask::Chat, "http://h/v1"), None); + assert_eq!( + task_endpoint_line(ModelTask::Unknown, "http://h/v1"), + None, + "no route to name" + ); + } +} diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 4e71cf58b5..0870dc5e2b 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -327,6 +327,7 @@ fn spawn_managed_engine_child( env_id: Option<&str>, engine_recipe: Option<&EngineRecipeHint>, require_api_key: bool, + task: rocm_core::ModelTask, ) -> Result { paths.ensure()?; fs::create_dir_all(paths.services_dir())?; @@ -415,6 +416,7 @@ fn spawn_managed_engine_child( ); record.gpu_indices = gpu_indices.to_vec(); record.engine_recipe_json = requested_recipe_json; + record.task = task; // The flag the user actually passed, carried through rather than re-derived. // // Deriving it from key-file presence looked equivalent and was not: @@ -451,6 +453,7 @@ fn spawn_managed_engine_child( engine_recipe, &record.engine_state_path, Some(&record.log_path), + task, )?; let engine_envs_root = env_root_for_service(paths, engine, runtime_id, env_id)?; // Hand the child the *path* to the endpoint key file (public bind only) via the @@ -552,6 +555,7 @@ pub(crate) fn start_managed_service( endpoint_api_key: Option<&str>, launch_lock: rocm_core::FileLock, require_api_key: bool, + task: rocm_core::ModelTask, on_wait_tick: &mut dyn FnMut(Duration), ) -> Result { let paths = AppPaths::discover()?; @@ -569,6 +573,7 @@ pub(crate) fn start_managed_service( env_id, engine_recipe, require_api_key, + task, )? { ManagedSpawn::AlreadyRunning(report) => return Ok(report), ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), @@ -583,6 +588,7 @@ pub(crate) fn start_managed_service( thread::sleep(Duration::from_millis(200)); let readiness = wait_for_service_http_ready_with_progress( + task, engine, host, port, @@ -629,12 +635,16 @@ pub(crate) fn start_managed_service( pub(crate) fn print_managed_launch_plain( report: &ManagedLaunchReport, endpoint_api_key: Option<&str>, + docs_link: Option<&str>, ) { if report.already_running { println!("managed service already running"); println!(" service_id: {}", report.service_id); println!(" endpoint: {}", report.endpoint_url); println!(" status: {}", report.status); + if let Some(link) = docs_link { + println!(" docs: {link}"); + } println!(" note: existing service detected; no second process spawned"); return; } @@ -664,6 +674,9 @@ pub(crate) fn print_managed_launch_plain( println!(" manifest_path: {}", manifest_path.display()); } println!(" readiness: {}", report.status); + if let Some(link) = docs_link { + println!(" docs: {link}"); + } } /// Render the one-time secure client configuration for a public, authenticated @@ -702,16 +715,21 @@ fn background_helper_already_running(paths: &AppPaths) -> Result { /// itself (`command.spawn()` / `spawn_detached_no_inherit`) is logged rather than /// propagated; setup errors (path discovery, stdio attach) still return `Err`. pub(crate) fn ensure_background_helper_running() -> Result<()> { - ensure_background_helper_running_quiet(false) + ensure_background_helper_running_quiet(false).map(drop) } /// As [`ensure_background_helper_running`], but suppresses the stdout status line /// when `quiet` is set. The interactive `rocm serve` summary path uses `quiet` so /// the daemon-spawn note does not appear above the deployment summary table. -pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> { +/// +/// Returns whether a helper is still starting up — spawned by this call or by a +/// concurrent one — as opposed to one that was already running (or could not be +/// spawned). Only a starting helper is worth waiting for before probing its API +/// docs. +pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result { let paths = AppPaths::discover()?; if background_helper_already_running(&paths)? { - return Ok(()); + return Ok(false); } // The check above and the spawn below are a TOCTOU window: two concurrent @@ -722,7 +740,7 @@ pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> // already-running case lock-free. let _autostart_lock = rocm_core::FileLock::acquire(paths.automation_autostart_lock_path())?; if background_helper_already_running(&paths)? { - return Ok(()); + return Ok(false); } // The lock alone does not close the window: the spawned daemon does not @@ -738,7 +756,7 @@ pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> AUTOSTART_CLAIM_TTL_MS, rocm_core::process_is_running, ) { - return Ok(()); + return Ok(true); } let exe = managed_service_launcher_path() @@ -759,6 +777,7 @@ pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> apply_app_path_env(&mut command, &paths); command.spawn().map(|child| child.id()) }; + let spawned = spawn_result.is_ok(); match spawn_result { Ok(daemon_pid) => { // Record the claim before returning (and thus releasing the lock) so a @@ -780,7 +799,7 @@ pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> } Err(_) => {} } - Ok(()) + Ok(spawned) } /// How long an autostart claim is honoured before it is treated as stale even if @@ -877,6 +896,7 @@ pub(crate) fn run_attached_service( endpoint_api_key: Option<&str>, launch_lock: rocm_core::FileLock, require_api_key: bool, + task: rocm_core::ModelTask, ) -> Result<()> { let paths = AppPaths::discover()?; @@ -894,6 +914,7 @@ pub(crate) fn run_attached_service( env_id, resolve.engine_recipe.as_ref(), require_api_key, + task, )?; // The claiming record is persisted (or an existing service was found), so the // selected GPU is now visible to concurrent auto-selection. Release the launch @@ -909,6 +930,14 @@ pub(crate) fn run_attached_service( println!(" service_id: {}", report.service_id); println!(" endpoint: {}", report.endpoint_url); println!(" status: {}", report.status); + if let Some(link) = crate::api_docs_link::service_docs_link( + &paths, + &report.service_id, + std::time::Duration::ZERO, + &mut || {}, + ) { + println!(" docs: {link}"); + } println!(" logs: rocm logs {}", report.service_id); println!( " stop: {}", @@ -928,14 +957,29 @@ pub(crate) fn run_attached_service( // server's record, and feeds the dashboard. Match the background ordering // (spawn, then ensure the helper) and keep it quiet so no status line breaks // into the log stream. - ensure_background_helper_running_quiet(true)?; + let helper_starting = ensure_background_helper_running_quiet(true)?; // The resolution detail (model, engine, runtime, GPU, warnings) was already // printed as the "serve plan" block in `serve()`; extend it with the launch // coordinates and the streaming hint rather than repeating it. let endpoint = format!("{}/v1", format_http_base_url(host, port)); + // Printed beside `endpoint`, never instead of it, so the base URL keeps one + // meaning on every surface; see `render_services_text`. + let task_endpoint_line = service_hints::task_endpoint_line(task, &endpoint); println!(" service_id: {service_id}"); println!(" endpoint: {endpoint}"); + if let Some(line) = &task_endpoint_line { + println!("{line}"); + } + let docs_link = crate::api_docs_link::service_docs_link( + &paths, + &service_id, + crate::api_docs_link::serve_docs_wait(helper_starting), + &mut || {}, + ); + if let Some(link) = docs_link.as_deref() { + println!(" docs: {link}"); + } if let Some(key) = endpoint_api_key { // Same intentional one-time key display as `print_managed_launch_plain` // above; see its rationale for why the tag below is currently inert. @@ -953,6 +997,12 @@ pub(crate) fn run_attached_service( println!("detached - server still running"); println!(" service_id: {service_id}"); println!(" endpoint: {endpoint}"); + if let Some(line) = &task_endpoint_line { + println!("{line}"); + } + if let Some(link) = docs_link.as_deref() { + println!(" docs: {link}"); + } println!(" list: rocm services"); println!(" logs: rocm logs {service_id}"); println!(" stop: {}", service_hints::service_stop_hint(&service_id)); @@ -1251,6 +1301,7 @@ mod tests { None, Some(&requested_recipe), false, + rocm_core::ModelTask::Chat, ); let _ = fs::remove_dir_all(root); @@ -1335,6 +1386,7 @@ mod tests { None, // The demand that used to be silently discarded. true, + rocm_core::ModelTask::Chat, ); let _ = fs::remove_dir_all(root); @@ -1420,6 +1472,7 @@ mod tests { None, None, true, + rocm_core::ModelTask::Chat, ); let _ = fs::remove_dir_all(root); diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 2f0064e18a..79fa551f78 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -16,6 +16,10 @@ e2e-test-hooks = ["rocm-core/e2e-test-hooks"] [dependencies] anyhow.workspace = true axum.workspace = true +# Forwards the API docs' try-it requests to the model server. Plain HTTP to +# loopback only, so no TLS stack; `stream` relays streamed completions as +# they arrive. +reqwest = { version = "0.12", default-features = false, features = ["stream"] } clap.workspace = true rocm-core = { path = "../../crates/rocm-core" } rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol" } @@ -24,6 +28,11 @@ serde_json.workspace = true sha2 = "0.10" tokio.workspace = true ureq = { version = "2.12", features = ["native-certs"] } +# API docs for running model servers: OpenAPI model + Swagger UI. `vendored` +# compiles the Swagger UI assets in, so the build downloads nothing and the +# page works offline. +utoipa = "6" +utoipa-swagger-ui = { version = "10", default-features = false, features = ["vendored"] } [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } diff --git a/apps/rocmd/src/api_docs.rs b/apps/rocmd/src/api_docs.rs new file mode 100644 index 0000000000..7757c39a48 --- /dev/null +++ b/apps/rocmd/src/api_docs.rs @@ -0,0 +1,1183 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! API docs for every running model server, served by the daemon on loopback. +//! +//! Of the servers rocm-cli runs, only vLLM generates its own docs; Lemonade and +//! the llama.cpp server behind it serve none. So the daemon describes each +//! running server from the routes rocm-cli knows it serves ([`routes_for`]), +//! renders that with Swagger UI, and forwards the page's "Try it out" requests +//! to the server through `/services//api/...`. Forwarding keeps try-it +//! same-origin, so it works on every engine whatever that engine's own +//! cross-origin settings are; Lemonade, for one, only admits browser requests +//! from its own origin. +//! +//! The server list is read from the service records on every request, so a +//! server started or stopped while the daemon runs needs no notification. + +use anyhow::{Context, Result, bail}; +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::{DefaultBodyLimit, Path, Request, State}; +use axum::http::{HeaderMap, HeaderValue, Method, StatusCode, Uri, header}; +use axum::middleware::{self, Next}; +use axum::response::{Html, IntoResponse, Redirect, Response}; +use axum::routing::{any, get}; +use rocm_core::ModelTask; +use rocm_core::{AppPaths, DEFAULT_LOCAL_HOST, ManagedServiceRecord}; +use serde_json::{Value, json}; +use std::fmt::Write as _; +use std::net::IpAddr; +use std::sync::Arc; +use std::time::Duration; +use tokio::net::TcpListener; +use utoipa::openapi::path::{HttpMethod, OperationBuilder, PathItem, PathsBuilder}; +use utoipa::openapi::request_body::RequestBodyBuilder; +use utoipa::openapi::schema::{KnownFormat, ObjectBuilder, SchemaFormat, Type}; +use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityRequirement, SecurityScheme}; +use utoipa::openapi::{ + ComponentsBuilder, ContentBuilder, InfoBuilder, OpenApi, OpenApiBuilder, Required, + ResponseBuilder, ServerBuilder, +}; + +/// Largest request body the try-it forwarder accepts. Generous enough for an +/// audio upload; the docs server is loopback-only, so this bounds memory, not +/// abuse from elsewhere. +const FORWARD_BODY_LIMIT_BYTES: usize = 64 * 1024 * 1024; + +/// How long the forwarder waits for the model server to accept a connection. +/// There is deliberately no overall timeout: a long completion may stream for +/// minutes. +const FORWARD_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +/// The OpenAPI security scheme name used for servers started with an endpoint +/// key; Swagger UI's "Authorize" box fills it in. +const ENDPOINT_KEY_SCHEME: &str = "endpoint_key"; + +/// A running docs server. It runs on a thread and runtime of its own: the +/// daemon's loop runs watcher checks synchronously on its single-threaded +/// runtime, and a docs page or a streamed try-it answer must not stall for +/// the length of a check. +pub(crate) struct ApiDocsServer { + pub(crate) base_url: String, + shutdown: tokio::sync::oneshot::Sender<()>, + thread: std::thread::JoinHandle<()>, +} + +impl ApiDocsServer { + /// Stop accepting requests and wait for the server thread to finish. + pub(crate) fn stop(self) { + let _ = self.shutdown.send(()); + let _ = self.thread.join(); + } +} + +#[derive(Clone)] +struct DocsState { + paths: Arc, + client: reqwest::Client, +} + +pub(crate) fn start_api_docs_server(paths: AppPaths, port: u16) -> Result { + let listener = std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, port)).with_context(|| { + format!("failed to bind API docs server to {DEFAULT_LOCAL_HOST}:{port}") + })?; + let addr = listener + .local_addr() + .context("failed to read API docs listener address")?; + if !addr.ip().is_loopback() { + bail!("API docs server must bind to loopback; resolved address was {addr}"); + } + listener + .set_nonblocking(true) + .context("failed to configure the API docs listener")?; + let app = router(DocsState { + paths: Arc::new(paths), + client: forwarding_client()?, + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("failed to build the API docs runtime")?; + let (shutdown, shutdown_received) = tokio::sync::oneshot::channel::<()>(); + let thread = std::thread::Builder::new() + .name("rocmd-api-docs".to_owned()) + .spawn(move || { + runtime.block_on(async move { + let served = match TcpListener::from_std(listener) { + Ok(listener) => axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = shutdown_received.await; + }) + .await + .map_err(anyhow::Error::from), + Err(error) => Err(error.into()), + }; + if let Err(error) = served { + eprintln!("API docs server stopped: {error}"); + } + }); + }) + .context("failed to start the API docs thread")?; + Ok(ApiDocsServer { + base_url: format!("http://{addr}"), + shutdown, + thread, + }) +} + +/// The client the forwarder sends try-it requests with. +fn forwarding_client() -> Result { + reqwest::Client::builder() + .connect_timeout(FORWARD_CONNECT_TIMEOUT) + // The model server is on this machine. A proxy configured for the + // user's outbound traffic (`HTTP_PROXY`, `ALL_PROXY`) would carry the + // prompt and the endpoint key off it, or fail the request outright. + .no_proxy() + // A redirect is not followed: the browser gets the model server's own + // status. The routes documented here never redirect. + .redirect(reqwest::redirect::Policy::none()) + .build() + .context("failed to build the API docs forwarding client") +} + +fn router(state: DocsState) -> Router { + Router::new() + .route("/", get(index_page)) + .route("/health", get(health)) + .route("/services/{id}/openapi.json", get(service_openapi_json)) + .route("/services/{id}/docs", get(docs_redirect)) + .route("/services/{id}/docs/", get(docs_index)) + .route("/services/{id}/docs/{*file}", get(docs_file)) + .route( + "/services/{id}/api/{*path}", + any(forward).layer(DefaultBodyLimit::max(FORWARD_BODY_LIMIT_BYTES)), + ) + .layer(middleware::from_fn(require_loopback_host)) + .with_state(state) +} + +/// Refuse requests whose `Host` is not a loopback name, and browser requests +/// sent from a page on another origin. The listener is already loopback-only; +/// the `Host` check closes DNS rebinding, where a web page resolves its own +/// hostname to 127.0.0.1, and the `Origin` check stops any other page the user +/// has open from posting to a model server through the forwarder. Requests +/// that are not from a browser carry no `Origin` and are not affected. +async fn require_loopback_host(request: Request, next: Next) -> Response { + let headers = request.headers(); + let host_allowed = headers + .get(header::HOST) + .and_then(|value| value.to_str().ok()) + .is_some_and(host_header_is_loopback); + let origin_allowed = headers + .get(header::ORIGIN) + .is_none_or(|value| value.to_str().is_ok_and(origin_is_loopback)); + if host_allowed && origin_allowed { + next.run(request).await + } else { + ( + StatusCode::FORBIDDEN, + "API docs answer only on 127.0.0.1 or localhost\n", + ) + .into_response() + } +} + +fn host_header_is_loopback(host: &str) -> bool { + let name = if let Some(rest) = host.strip_prefix('[') { + rest.split(']').next().unwrap_or_default() + } else { + host.rsplit_once(':').map_or(host, |(name, _)| name) + }; + name.eq_ignore_ascii_case("localhost") + || name.parse::().is_ok_and(|ip| ip.is_loopback()) +} + +/// Whether an `Origin` header names a page served from this machine. +fn origin_is_loopback(origin: &str) -> bool { + origin + .strip_prefix("http://") + .or_else(|| origin.strip_prefix("https://")) + .is_some_and(|authority| !authority.contains('/') && host_header_is_loopback(authority)) +} + +async fn health() -> impl IntoResponse { + axum::Json(json!({ "status": "ok", "source": "api_docs", "bind": "loopback_only" })) +} + +/// Every server the record says is up. Errors reading the records read as "no +/// servers": the docs page must never take the daemon down. +fn running_services(paths: &AppPaths) -> Vec { + crate::persistence::load_managed_services(paths) + .unwrap_or_default() + .into_iter() + .filter(service_is_running) + .collect() +} + +fn service_is_running(record: &ManagedServiceRecord) -> bool { + matches!( + record.status.as_str(), + "starting" | "running" | "recovering" | "ready" + ) +} + +/// The running server with this id. Looked up by matching the id against the +/// records rather than by building a path from it, so a crafted id cannot +/// reach outside the services folder. +fn find_running_service(paths: &AppPaths, service_id: &str) -> Option { + running_services(paths) + .into_iter() + .find(|record| record.service_id == service_id) +} + +fn not_running_page(service_id: &str) -> Response { + let id = html_escape(service_id); + ( + StatusCode::NOT_FOUND, + Html(format!( + "Server not running\ +

Server {id} is not running

\ +

Its API docs are available while it runs. \ + Running servers

" + )), + ) + .into_response() +} + +async fn index_page(State(state): State) -> Html { + let services = running_services(&state.paths); + let mut rows = String::new(); + for record in &services { + let id = html_escape(&record.service_id); + let _ = write!( + rows, + "
  • {model} ({engine}, {id}, {endpoint})
  • ", + model = html_escape(&record.canonical_model_id), + engine = html_escape(&record.engine), + endpoint = html_escape(&record.endpoint_url), + ); + } + let body = if services.is_empty() { + "

    No model server is running. Start one with rocm serve.

    ".to_owned() + } else { + format!("
      {rows}
    ") + }; + Html(format!( + "ROCm CLI API docs\ +

    Running model servers

    {body}" + )) +} + +async fn service_openapi_json( + State(state): State, + Path(service_id): Path, +) -> Response { + match find_running_service(&state.paths, &service_id) { + Some(record) => axum::Json(service_openapi(&record)).into_response(), + None => ( + StatusCode::NOT_FOUND, + axum::Json(json!({ + "error": format!("server {service_id} is not running"), + })), + ) + .into_response(), + } +} + +async fn docs_redirect(Path(service_id): Path) -> Redirect { + Redirect::permanent(&format!("/services/{service_id}/docs/")) +} + +async fn docs_index(State(state): State, Path(service_id): Path) -> Response { + serve_swagger_file(&state, &service_id, "") +} + +async fn docs_file( + State(state): State, + Path((service_id, file)): Path<(String, String)>, +) -> Response { + serve_swagger_file(&state, &service_id, &file) +} + +fn serve_swagger_file(state: &DocsState, service_id: &str, file: &str) -> Response { + if find_running_service(&state.paths, service_id).is_none() { + return not_running_page(service_id); + } + let config = Arc::new( + utoipa_swagger_ui::Config::new([format!("/services/{service_id}/openapi.json")]) + // The endpoint key typed into "Authorize" lasts until the page is + // closed; it is not saved in the browser, where it would outlive + // the server it belongs to. + .try_it_out_enabled(true), + ); + match utoipa_swagger_ui::serve(file, config) { + Ok(Some(swagger_file)) => ( + [(header::CONTENT_TYPE, swagger_file.content_type)], + swagger_file.bytes.into_owned(), + ) + .into_response(), + Ok(None) => StatusCode::NOT_FOUND.into_response(), + Err(error) => (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(), + } +} + +/// Relay one try-it request to the model server and stream its answer back. +async fn forward( + State(state): State, + Path((service_id, _decoded_path)): Path<(String, String)>, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let Some(record) = find_running_service(&state.paths, &service_id) else { + return ( + StatusCode::NOT_FOUND, + axum::Json(json!({ "error": format!("server {service_id} is not running") })), + ) + .into_response(); + }; + // The path exactly as the browser sent it. The extracted one is + // percent-decoded: an encoded `?` or `/` in it would turn into a query or + // an extra segment, and the request would reach a different route. + let Some(path) = raw_forwarded_path(uri.path()) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let mut target = format!("{}/{path}", service_base_url(&record)); + if let Some(query) = uri.query() { + target.push('?'); + target.push_str(query); + } + let mut request = state.client.request(method, &target).body(body); + // Only what the routes need: the body's type, the answer the caller + // accepts, and the endpoint key the user typed into "Authorize". Nothing + // the browser adds on its own (cookies, origin) reaches the model server. + for name in [header::CONTENT_TYPE, header::ACCEPT, header::AUTHORIZATION] { + if let Some(value) = headers.get(&name) { + request = request.header(name, value); + } + } + match request.send().await { + Ok(answer) => { + let status = answer.status(); + let content_type = answer.headers().get(header::CONTENT_TYPE).cloned(); + let mut response = Response::new(Body::from_stream(answer.bytes_stream())); + *response.status_mut() = status; + if let Some(content_type) = content_type { + response + .headers_mut() + .insert(header::CONTENT_TYPE, content_type); + } + response + } + Err(error) => ( + StatusCode::BAD_GATEWAY, + [( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + )], + json!({ + "error": format!( + "could not reach server {service_id} at {}: {error}", + service_base_url(&record) + ), + }) + .to_string(), + ) + .into_response(), + } +} + +/// The part of `/services//api/` after `api/`, still percent-encoded. +fn raw_forwarded_path(request_path: &str) -> Option<&str> { + let mut segments = request_path.splitn(5, '/'); + match ( + segments.next(), + segments.next(), + segments.next(), + segments.next(), + segments.next(), + ) { + (Some(""), Some("services"), Some(_), Some("api"), Some(path)) => Some(path), + _ => None, + } +} + +/// Where the forwarder connects. A server bound to every interface is reached +/// on loopback, the same address rocm-cli's own readiness checks use. +fn service_base_url(record: &ManagedServiceRecord) -> String { + let host = match record.host.parse::() { + Ok(ip) if ip.is_unspecified() => DEFAULT_LOCAL_HOST, + _ => record.host.as_str(), + }; + rocm_core::format_http_base_url(host, record.port) +} + +/// One route a model server answers, as the docs describe it. +struct RouteDoc { + method: HttpMethod, + path: &'static str, + summary: &'static str, + description: &'static str, + body: RouteBody, +} + +/// What a route's request carries, as "Try it out" should offer it. +enum RouteBody { + None, + /// JSON, with an example built for the server's real model id. + Json(fn(&str) -> Value), + /// A multipart audio upload, as the OpenAI transcription API takes it. + AudioUpload, +} + +/// The routes each engine's server answers: its health route plus +/// [`rocm_engine_protocol::task_routes`] for the server's task, the set its +/// plugin reports as `supported_routes`. The single place the docs learn what a +/// server serves; a new kind of server extends this. +fn routes_for(record: &ManagedServiceRecord) -> Vec { + let health_path = if record.engine == "vllm" { + "/health" + } else { + "/v1/health" + }; + let mut routes = vec![ + RouteDoc { + method: HttpMethod::Get, + path: health_path, + summary: "Check the server is up", + description: "Answers 200 once the server accepts requests.", + body: RouteBody::None, + }, + RouteDoc { + method: HttpMethod::Get, + path: "/v1/models", + summary: "List the served models", + description: "OpenAI-compatible model list. The `id` of each entry is the \ + `model` value requests use.", + body: RouteBody::None, + }, + ]; + match record.task { + ModelTask::Chat => routes.extend(chat_routes()), + ModelTask::Transcription => routes.push(RouteDoc { + method: HttpMethod::Post, + path: "/v1/audio/transcriptions", + summary: "Transcribe speech", + description: "OpenAI-compatible speech-to-text. Upload an audio file as `file` \ + (WAV is accepted by every backend); the answer's `text` is the \ + transcript.", + body: RouteBody::AudioUpload, + }), + // A task this build does not know: only the routes every server has. + ModelTask::Unknown => {} + } + routes +} + +fn chat_routes() -> [RouteDoc; 2] { + [ + RouteDoc { + method: HttpMethod::Post, + path: "/v1/chat/completions", + summary: "Chat completion", + description: "OpenAI-compatible chat completion. Set `stream` to `true` to \ + receive server-sent events as tokens are generated.", + body: RouteBody::Json(chat_completion_example), + }, + RouteDoc { + method: HttpMethod::Post, + path: "/v1/completions", + summary: "Text completion", + description: "OpenAI-compatible text completion of a raw prompt.", + body: RouteBody::Json(completion_example), + }, + ] +} + +/// The form "Try it out" shows for a transcription: a file picker for the +/// audio and the model field already filled in. +fn audio_upload_schema(model: &str) -> ObjectBuilder { + ObjectBuilder::new() + .property( + "file", + ObjectBuilder::new() + .schema_type(Type::String) + .format(Some(SchemaFormat::KnownFormat(KnownFormat::Binary))) + .description(Some("The audio to transcribe, for example a WAV file.")), + ) + .required("file") + .property( + "model", + ObjectBuilder::new() + .schema_type(Type::String) + .default(Some(json!(model))), + ) + .required("model") +} + +fn chat_completion_example(model: &str) -> Value { + json!({ + "model": model, + "messages": [{ "role": "user", "content": "Say hello in one sentence." }], + "max_tokens": 64, + "stream": false, + }) +} + +fn completion_example(model: &str) -> Value { + json!({ + "model": model, + "prompt": "The capital of France is", + "max_tokens": 16, + }) +} + +/// Where each engine documents more than rocm-cli describes. +fn engine_docs_note(record: &ManagedServiceRecord) -> String { + let base = service_base_url(record); + if record.engine == "vllm" { + format!( + "vLLM also documents every route it serves, beyond the ones listed here, at \ + [{base}/docs]({base}/docs)." + ) + } else if !record.task.is_chat() { + String::new() + } else { + "On Linux, Lemonade runs chat models with the llama.cpp server, which also answers its \ + own routes; see the \ + [llama.cpp server documentation](https://github.com/ggml-org/llama.cpp/tree/master/tools/server)." + .to_owned() + } +} + +fn service_openapi(record: &ManagedServiceRecord) -> OpenApi { + let model = record.canonical_model_id.as_str(); + let mut paths = PathsBuilder::new(); + for route in routes_for(record) { + let mut operation = OperationBuilder::new() + .summary(Some(route.summary)) + .description(Some(route.description)) + .response( + "200", + ResponseBuilder::new() + .description("The server's answer") + .content("application/json", ContentBuilder::new().build()), + ); + let request_body = match route.body { + RouteBody::None => None, + RouteBody::Json(example_body) => Some( + RequestBodyBuilder::new() + .required(Some(Required::True)) + .content( + "application/json", + ContentBuilder::new() + .example(Some(example_body(model))) + .build(), + ) + .build(), + ), + RouteBody::AudioUpload => Some( + RequestBodyBuilder::new() + .required(Some(Required::True)) + .content( + "multipart/form-data", + ContentBuilder::new() + .schema(Some(audio_upload_schema(model))) + .build(), + ) + .build(), + ), + }; + operation = operation.request_body(request_body); + paths = paths.path(route.path, PathItem::new(route.method, operation)); + } + + let description = format!( + "`{model}` ({task}) served by {engine} at `{endpoint}`.\n\n\ + \"Try it out\" sends requests through this page to the server.\n\n{note}", + task = record.task.display_name(), + engine = record.engine, + endpoint = record.endpoint_url, + note = engine_docs_note(record), + ); + let mut builder = OpenApiBuilder::new() + .info( + InfoBuilder::new() + .title(format!("{model} ({})", record.service_id)) + .version(env!("CARGO_PKG_VERSION")) + .description(Some(description)), + ) + .servers(Some([ServerBuilder::new() + .url(format!("/services/{}/api", record.service_id)) + .description(Some(format!( + "{} through the docs page", + record.endpoint_url + ))) + .build()])) + .paths(paths); + if record.requires_api_key { + builder = builder + .components(Some( + ComponentsBuilder::new() + .security_scheme( + ENDPOINT_KEY_SCHEME, + SecurityScheme::Http( + HttpBuilder::new().scheme(HttpAuthScheme::Bearer).build(), + ), + ) + .build(), + )) + .security(Some([SecurityRequirement::new( + ENDPOINT_KEY_SCHEME, + Vec::::new(), + )])); + } + builder.build() +} + +fn html_escape(text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + match character { + '&' => escaped.push_str("&"), + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + '\'' => escaped.push_str("'"), + other => escaped.push(other), + } + } + escaped +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::temp_app_paths; + use axum::Json; + use axum::routing::post; + use rocm_core::RocmCliConfig; + use std::fs; + use std::sync::Mutex; + + /// Serializes the tests that set proxy variables. Every other test here + /// talks to its servers through [`test_client`], which ignores them. + static PROXY_ENV_TEST_LOCK: Mutex<()> = Mutex::new(()); + + /// The test's own HTTP client: never proxied, so a proxy variable set by + /// the proxy test cannot reroute it. + fn test_client() -> reqwest::Client { + reqwest::Client::builder().no_proxy().build().unwrap() + } + + fn record(paths: &AppPaths, service_id: &str, engine: &str, port: u16) -> ManagedServiceRecord { + let mut record = ManagedServiceRecord::new( + paths, + service_id, + engine, + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record + } + + fn operation_paths(openapi: &Value) -> Vec { + openapi["paths"] + .as_object() + .expect("paths object") + .keys() + .cloned() + .collect() + } + + #[test] + fn chat_server_docs_list_its_routes_with_the_served_model() { + let (root, paths) = temp_app_paths("api-docs-chat-routes"); + for (engine, health) in [("vllm", "/health"), ("lemonade", "/v1/health")] { + let openapi = + serde_json::to_value(service_openapi(&record(&paths, "svc-1", engine, 9))).unwrap(); + let mut listed = operation_paths(&openapi); + listed.sort(); + let mut expected = vec![ + health.to_owned(), + "/v1/chat/completions".to_owned(), + "/v1/completions".to_owned(), + "/v1/models".to_owned(), + ]; + expected.sort(); + assert_eq!(listed, expected, "{engine}"); + assert_eq!( + openapi["paths"]["/v1/chat/completions"]["post"]["requestBody"]["content"]["application/json"] + ["example"]["model"], + "Qwen/Qwen3.5", + "{engine}: try-it must start from the model the server answers to" + ); + assert_eq!( + openapi["servers"][0]["url"], "/services/svc-1/api", + "{engine}" + ); + assert!( + openapi.get("security").is_none(), + "{engine}: loopback needs no key" + ); + } + fs::remove_dir_all(root).ok(); + } + + #[test] + fn documented_routes_are_the_routes_the_engines_report() { + let (root, paths) = temp_app_paths("api-docs-protocol-routes"); + for (engine, health) in [("vllm", "/health"), ("lemonade", "/v1/health")] { + for task in [ + ModelTask::Chat, + ModelTask::Transcription, + ModelTask::Unknown, + ] { + let mut server = record(&paths, "svc-1", engine, 9); + server.task = task; + let documented: Vec = routes_for(&server) + .iter() + .map(|route| route.path.to_owned()) + .collect(); + assert_eq!( + documented, + rocm_engine_protocol::supported_routes(health, task), + "{engine} {task}" + ); + } + } + fs::remove_dir_all(root).ok(); + } + + #[test] + fn forwarded_paths_keep_their_encoding() { + assert_eq!( + raw_forwarded_path("/services/svc-1/api/v1/models/Qwen%2FQwen3.5%3Fx"), + Some("v1/models/Qwen%2FQwen3.5%3Fx") + ); + assert_eq!(raw_forwarded_path("/services/svc-1/api/"), Some("")); + assert_eq!(raw_forwarded_path("/services/svc-1/docs/x"), None); + } + + #[test] + fn only_pages_from_this_machine_may_send_browser_requests() { + for origin in [ + "http://127.0.0.1:11430", + "http://localhost:11430", + "http://[::1]:11430", + ] { + assert!(origin_is_loopback(origin), "{origin}"); + } + for origin in [ + "https://evil.example", + "http://127.0.0.1.evil.example", + "null", + "", + ] { + assert!(!origin_is_loopback(origin), "{origin}"); + } + } + + #[test] + fn the_bundled_swagger_ui_is_the_version_named_in_the_notices() { + let config = Arc::new(utoipa_swagger_ui::Config::new(["/openapi.json"])); + let bundle = utoipa_swagger_ui::serve("swagger-ui-bundle.js", config) + .unwrap() + .expect("the vendored Swagger UI ships its bundle"); + let bundle = String::from_utf8_lossy(&bundle.bytes); + let marker = "PACKAGE_VERSION:\""; + let start = bundle.find(marker).expect("the bundle states its version") + marker.len(); + let version: String = bundle[start..].chars().take_while(|c| *c != '"').collect(); + // Read, not `include_str!`: the file is not ASCII, and xtask's output + // contract rejects non-ASCII data compiled into a binary. + let notices = fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../THIRD_PARTY_NOTICES.txt" + )) + .expect("THIRD_PARTY_NOTICES.txt at the repo root"); + assert!( + notices.contains(&format!(" - Swagger UI {version} ")), + "THIRD_PARTY_NOTICES.txt must name the Swagger UI {version} that rocmd serves; \ + update the bundled-assets section of about.hbs and run `cargo xtask tpn`" + ); + } + + #[test] + fn docs_stay_off_without_stopping_the_daemon() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-off"); + paths.ensure()?; + let mut state = crate::service::build_runtime_state(&RocmCliConfig::default(), true); + + let mut disabled = RocmCliConfig::default(); + disabled.api_docs.enabled = false; + assert!(crate::service::start_api_docs(&paths, &disabled, &mut state)?.is_none()); + + let mut port_zero = RocmCliConfig::default(); + port_zero.api_docs.port = 0; + assert!(crate::service::start_api_docs(&paths, &port_zero, &mut state)?.is_none()); + + let taken = std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, 0))?; + let mut on_taken_port = RocmCliConfig::default(); + on_taken_port.api_docs.port = taken.local_addr()?.port(); + assert!( + crate::service::start_api_docs(&paths, &on_taken_port, &mut state)?.is_none(), + "a taken port leaves docs off instead of failing the daemon" + ); + let events = fs::read_to_string(paths.automation_events_path())?; + assert!(events.contains("api_docs_unavailable"), "{events}"); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[test] + fn speech_to_text_docs_offer_an_audio_upload_and_no_chat_routes() { + let (root, paths) = temp_app_paths("api-docs-transcription"); + let mut whisper = record(&paths, "svc-asr", "lemonade", 9); + whisper.task = ModelTask::Transcription; + whisper.canonical_model_id = "Whisper-Large-v3-Turbo".to_owned(); + let openapi = serde_json::to_value(service_openapi(&whisper)).unwrap(); + let mut listed = operation_paths(&openapi); + listed.sort(); + assert_eq!( + listed, + ["/v1/audio/transcriptions", "/v1/health", "/v1/models"] + ); + let form = &openapi["paths"]["/v1/audio/transcriptions"]["post"]["requestBody"]["content"] + ["multipart/form-data"]["schema"]; + assert_eq!(form["properties"]["file"]["format"], "binary", "{form:#}"); + assert_eq!( + form["properties"]["model"]["default"], "Whisper-Large-v3-Turbo", + "try-it must start from the model the server answers to" + ); + assert!( + !openapi["info"]["description"] + .as_str() + .unwrap_or_default() + .contains("llama.cpp"), + "the llama.cpp note is about chat models" + ); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn keyed_server_docs_ask_for_the_endpoint_key() { + let (root, paths) = temp_app_paths("api-docs-keyed"); + let mut keyed = record(&paths, "svc-1", "vllm", 9); + keyed.requires_api_key = true; + let openapi = serde_json::to_value(service_openapi(&keyed)).unwrap(); + assert_eq!( + openapi["components"]["securitySchemes"][ENDPOINT_KEY_SCHEME]["scheme"], + "bearer" + ); + assert!(openapi["security"][0].get(ENDPOINT_KEY_SCHEME).is_some()); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn engine_note_links_vllm_docs_and_llama_cpp_routes() { + let (root, paths) = temp_app_paths("api-docs-note"); + assert!( + engine_docs_note(&record(&paths, "a", "vllm", 8000)) + .contains("http://127.0.0.1:8000/docs") + ); + assert!(engine_docs_note(&record(&paths, "b", "lemonade", 8000)).contains("llama.cpp")); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn a_server_bound_to_every_interface_is_reached_on_loopback() { + let (root, paths) = temp_app_paths("api-docs-wildcard"); + let mut wildcard = record(&paths, "a", "vllm", 8000); + wildcard.host = "0.0.0.0".to_owned(); + assert_eq!(service_base_url(&wildcard), "http://127.0.0.1:8000"); + wildcard.host = "::".to_owned(); + assert_eq!(service_base_url(&wildcard), "http://127.0.0.1:8000"); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn only_loopback_host_names_are_accepted() { + for host in [ + "127.0.0.1:11430", + "localhost:11430", + "LOCALHOST", + "[::1]:11430", + "127.0.0.1", + ] { + assert!(host_header_is_loopback(host), "{host}"); + } + for host in [ + "evil.example:11430", + "192.168.1.5:11430", + "[2001:db8::1]:11430", + "", + ] { + assert!(!host_header_is_loopback(host), "{host}"); + } + } + + #[test] + fn html_escape_neutralises_markup() { + assert_eq!( + html_escape("'&'"), + "<a href="x">'&'</a>" + ); + } + + /// A stand-in model server that echoes what reached it, so the test can + /// see exactly what the forwarder passed on. + async fn start_echo_model_server() -> u16 { + let listener = TcpListener::bind((DEFAULT_LOCAL_HOST, 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let app = Router::new().route( + "/v1/chat/completions", + post(|headers: HeaderMap, uri: Uri, body: Bytes| async move { + Json(json!({ + "query": uri.query(), + "authorization": headers + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()), + "cookie": headers.get(header::COOKIE).and_then(|value| value.to_str().ok()), + "body": String::from_utf8_lossy(&body), + })) + }), + ); + tokio::spawn(async move { axum::serve(listener, app).await.ok() }); + port + } + + #[tokio::test(flavor = "multi_thread")] + async fn docs_server_describes_and_forwards_to_a_running_server() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-live"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let model_port = start_echo_model_server().await; + record(&paths, "svc-live", "vllm", model_port).write()?; + let mut stopped = record(&paths, "svc-stopped", "vllm", model_port); + stopped.status = "stopped".to_owned(); + stopped.write()?; + + let server = start_api_docs_server(paths.clone(), 0)?; + let base = server.base_url.clone(); + let client = test_client(); + + let index = client.get(format!("{base}/")).send().await?.text().await?; + assert!(index.contains("/services/svc-live/docs/"), "{index}"); + assert!(!index.contains("svc-stopped"), "{index}"); + + let page = client + .get(format!("{base}/services/svc-live/docs/")) + .send() + .await?; + assert_eq!(page.status(), StatusCode::OK); + assert!(page.text().await?.contains("swagger-ui")); + let initializer = client + .get(format!( + "{base}/services/svc-live/docs/swagger-initializer.js" + )) + .send() + .await? + .text() + .await?; + assert!( + initializer.contains("/services/svc-live/openapi.json"), + "Swagger UI must load this server's description: {initializer}" + ); + + let openapi: Value = client + .get(format!("{base}/services/svc-live/openapi.json")) + .send() + .await? + .text() + .await? + .parse()?; + assert_eq!(openapi["servers"][0]["url"], "/services/svc-live/api"); + + let forwarded: Value = client + .post(format!( + "{base}/services/svc-live/api/v1/chat/completions?trace=1" + )) + .header(header::AUTHORIZATION, "Bearer typed-in-docs") + .header(header::COOKIE, "session=browser-only") + .header(header::CONTENT_TYPE, "application/json") + .body(r#"{"model":"Qwen/Qwen3.5"}"#) + .send() + .await? + .text() + .await? + .parse()?; + assert_eq!(forwarded["body"], r#"{"model":"Qwen/Qwen3.5"}"#); + assert_eq!(forwarded["query"], "trace=1"); + assert_eq!(forwarded["authorization"], "Bearer typed-in-docs"); + assert_eq!( + forwarded["cookie"], + Value::Null, + "browser cookies stay in the browser" + ); + + for path in ["docs/", "openapi.json", "api/v1/models"] { + let response = client + .get(format!("{base}/services/svc-stopped/{path}")) + .send() + .await?; + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{path}"); + } + let stopped_page = client + .get(format!("{base}/services/svc-stopped/docs/")) + .send() + .await? + .text() + .await?; + assert!(stopped_page.contains("is not running"), "{stopped_page}"); + + let rebound = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::HOST, "attacker.example") + .body("{}") + .send() + .await?; + assert_eq!(rebound.status(), StatusCode::FORBIDDEN, "DNS rebinding"); + let cross_site = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::ORIGIN, "https://some-site.example") + .header(header::CONTENT_TYPE, "text/plain") + .body("{}") + .send() + .await?; + assert_eq!( + cross_site.status(), + StatusCode::FORBIDDEN, + "cross-site post" + ); + let same_origin = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::ORIGIN, base.as_str()) + .body("{}") + .send() + .await?; + assert_eq!(same_origin.status(), StatusCode::OK, "the docs page itself"); + + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// Restores a proxy variable when the test ends, even by panicking. + struct RestoredProxyVar(&'static str, Option); + + impl Drop for RestoredProxyVar { + #[allow(unsafe_code)] // std::env::set_var/remove_var are unsafe in edition 2024 + fn drop(&mut self) { + // SAFETY: still inside PROXY_ENV_TEST_LOCK, which the test holds + // for longer than this guard lives. + unsafe { + match &self.1 { + Some(value) => std::env::set_var(self.0, value), + None => std::env::remove_var(self.0), + } + } + } + } + + // A plain test driving its own runtime, so the std lock guarding the + // process environment is never held across an `.await`. + #[allow(unsafe_code)] // std::env::set_var is unsafe in edition 2024 + #[test] + fn try_it_reaches_the_server_even_with_a_proxy_configured() -> Result<()> { + let _lock = PROXY_ENV_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + let (root, paths) = temp_app_paths("api-docs-proxy"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let model_port = runtime.block_on(start_echo_model_server()); + record(&paths, "svc-live", "vllm", model_port).write()?; + // A proxy that is not there: a request sent to it fails. + let dead_proxy = format!( + "http://127.0.0.1:{}", + std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, 0))? + .local_addr()? + .port() + ); + // Every original value is saved before any is changed: on Windows + // `HTTP_PROXY` and `http_proxy` name one variable, so saving one after + // setting the other would save the dead proxy as the "original". + let proxied = ["HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy"]; + let unproxied = ["NO_PROXY", "no_proxy"]; + let _restore: Vec = proxied + .iter() + .chain(&unproxied) + .map(|key| RestoredProxyVar(key, std::env::var_os(key))) + .collect(); + for key in proxied { + // SAFETY: PROXY_ENV_TEST_LOCK is held; `_restore` puts the old + // values back before it is released. + unsafe { std::env::set_var(key, &dead_proxy) }; + } + for key in unproxied { + // SAFETY: as above. + unsafe { std::env::remove_var(key) }; + } + + let server = start_api_docs_server(paths, 0)?; + let status = runtime.block_on(async { + test_client() + .post(format!( + "{}/services/svc-live/api/v1/chat/completions", + server.base_url + )) + .body("{}") + .send() + .await + .map(|response| response.status()) + })?; + assert_eq!(status, StatusCode::OK, "try-it went to the proxy"); + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn forwarding_to_a_dead_server_reports_a_bad_gateway() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-dead"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + // Bind and drop to get a port nothing listens on. + let dead_port = std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, 0))? + .local_addr()? + .port(); + record(&paths, "svc-dead", "lemonade", dead_port).write()?; + let server = start_api_docs_server(paths.clone(), 0)?; + let response = test_client() + .get(format!( + "{}/services/svc-dead/api/v1/models", + server.base_url + )) + .send() + .await?; + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert!( + response + .text() + .await? + .contains("could not reach server svc-dead") + ); + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } +} diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index 3333461903..d442e53138 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -4,6 +4,7 @@ #![allow(clippy::items_after_test_module)] +mod api_docs; mod cli; mod common; mod mcp; diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 1a8c3335e9..eec7793f57 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -351,6 +351,40 @@ fn terminate_recorded_pid(identity: &rocm_core::ProcessIdentity) -> rocm_core::T ) } +/// Start the API docs server for running model servers, unless `config.json` +/// turns it off. A taken port leaves docs off and is recorded as an +/// `api_docs_unavailable` automation event; the daemon's own work carries on +/// either way. +pub(crate) fn start_api_docs( + paths: &AppPaths, + config: &RocmCliConfig, + state: &mut AutomationRuntimeState, +) -> Result> { + let Some(port) = config.api_docs.served_port() else { + println!(" api_docs: disabled"); + return Ok(None); + }; + match crate::api_docs::start_api_docs_server(paths.clone(), port) { + Ok(server) => { + println!(" api_docs: {}", server.base_url); + Ok(Some(server)) + } + Err(error) => { + println!(" api_docs: off ({error:#})"); + crate::persistence::record_event( + paths, + state, + "rocmd", + "warn", + "api_docs_unavailable", + &format!("API docs for running model servers are off: {error:#}"), + None, + )?; + Ok(None) + } + } +} + pub(crate) async fn run_daemon( paths: &AppPaths, automations_enabled: bool, @@ -415,6 +449,10 @@ pub(crate) async fn run_daemon( } paths.ensure()?; + // Before the first state write: that write is what tells `rocm serve` a + // helper is already running, and a serve that reads it then probes the + // docs once, without waiting for them. + let api_docs = start_api_docs(paths, &config, &mut state)?; state.write(paths)?; crate::persistence::record_event( paths, @@ -517,6 +555,9 @@ pub(crate) async fn run_daemon( if let Some(task) = local_webhook_task { task.abort(); } + if let Some(server) = api_docs { + server.stop(); + } Ok(()) } @@ -671,7 +712,22 @@ pub(crate) fn supervise_service( // set by a stop issued since — and `ManagedServiceRecord::new` starting it // at `None` would quietly withdraw that stop and hand the service straight // back to the daemon's recovery. - record.stop_requested_unix_ms = existing.and_then(|existing| existing.stop_requested_unix_ms); + record.stop_requested_unix_ms = existing + .as_ref() + .and_then(|existing| existing.stop_requested_unix_ms); + // The task is carried over for the same reason: `new` starts every record + // as chat, and a recovered speech-to-text service relaunched and written + // back as chat would be loaded through the chat path and offered to chat + // clients from then on. The registry is the only place the task is kept + // between runs, so a record that is not there yet is a chat service. + record.task = existing.map(|existing| existing.task).unwrap_or_default(); + // Before the manifest write, so an unrecoverable record keeps the task a + // newer rocm-cli wrote instead of being relaunched with one this build + // cannot name. + record + .task + .ensure_servable() + .with_context(|| format!("not recovering `{}`", record.service_id))?; // Refuse a keyless public respawn before the manifest write, so a refused // attempt leaves the recorded restart_count and timestamps intact instead of // clobbering them with a record no live process will ever back. The spawn @@ -707,6 +763,7 @@ pub(crate) fn supervise_service( env_id.as_deref(), engine_recipe_json.as_deref(), &record.engine_state_path, + record.task, )) .stdin(Stdio::null()) .stdout(Stdio::from(log_file)) @@ -836,6 +893,7 @@ fn engine_serve_http_args( env_id: Option<&str>, engine_recipe_json: Option<&str>, state_path: &Path, + task: rocm_core::ModelTask, ) -> Vec { let mut args = vec![ "__engine-serve-http".to_owned(), @@ -859,10 +917,15 @@ fn engine_serve_http_args( engine_recipe_json, )); args.extend(["--state-path".to_owned(), state_path.display().to_string()]); + // Omitted for chat so a chat restart keeps the argv it always had; a + // speech-to-text service must come back as one. + if !task.is_chat() { + args.extend(["--task".to_owned(), task.as_str().to_owned()]); + } args } -fn build_runtime_state( +pub(crate) fn build_runtime_state( config: &RocmCliConfig, automations_enabled: bool, ) -> AutomationRuntimeState { @@ -1325,6 +1388,7 @@ mod tests { Some("env-1"), Some(engine_recipe_json), Path::new("state.json"), + rocm_core::ModelTask::Chat, ); assert!( @@ -1356,6 +1420,7 @@ mod tests { None, None, Path::new("state.json"), + rocm_core::ModelTask::Chat, ); assert!( @@ -1375,6 +1440,7 @@ mod tests { None, None, Path::new("state.json"), + rocm_core::ModelTask::Chat, ); assert!(!auto.iter().any(|arg| arg == "--gpu")); } @@ -2128,4 +2194,117 @@ mod tests { assert_eq!(persisted?.stop_requested_unix_ms, Some(1)); Ok(()) } + + /// Recovery rebuilds the record from its arguments, and those carry no + /// task: the one on disk must survive, or a recovered speech-to-text + /// service is relaunched and recorded as a chat service. + #[cfg(target_os = "linux")] + #[test] + fn supervise_service_first_write_keeps_a_speech_to_text_task() -> Result<()> { + let (root, paths) = temp_app_paths("supervise-first-write-keeps-task"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-supervise-first-write-task"; + let mut seeded = identity_probe_record(&paths, service_id, 11449); + seeded.task = rocm_core::ModelTask::Transcription; + seeded.write()?; + // Stop `supervise_service` right after its first write; see + // `supervise_service_persists_the_supervisor_identity_token`. + fs::create_dir_all(&seeded.log_path)?; + + let outcome = supervise_service( + &paths, + service_id.to_owned(), + "lemonade".to_owned(), + "Whisper-Tiny".to_owned(), + "Whisper-Tiny".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11449, + "gpu_required".to_owned(), + None, + None, + ); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert!(outcome.is_err(), "the log path is a directory"); + assert_eq!(persisted?.task, rocm_core::ModelTask::Transcription); + Ok(()) + } + + #[test] + fn supervise_service_does_not_recover_a_task_from_a_newer_rocm_cli() -> Result<()> { + let (root, paths) = temp_app_paths("supervise-unknown-task"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-supervise-unknown-task"; + let mut seeded = identity_probe_record(&paths, service_id, 11450); + seeded.task = rocm_core::ModelTask::Unknown; + seeded.restart_count = 3; + seeded.write()?; + + let outcome = supervise_service( + &paths, + service_id.to_owned(), + "lemonade".to_owned(), + "future-model".to_owned(), + "future-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11450, + "gpu_required".to_owned(), + None, + None, + ); + let persisted = load_service_record(&paths, service_id)?; + let log_created = seeded.log_path.exists(); + fs::remove_dir_all(root).ok(); + + let error = format!( + "{:#}", + outcome.expect_err("an unknown task is not recovered") + ); + assert!(error.contains("not recovering"), "{error}"); + assert!(error.contains("a newer rocm-cli"), "{error}"); + // Refused before the record was rewritten or anything launched. + assert_eq!(persisted.task, rocm_core::ModelTask::Unknown); + assert_eq!(persisted.restart_count, 3); + assert!(!log_created, "no engine was started"); + Ok(()) + } + + #[test] + fn a_speech_to_text_restart_carries_its_task_to_the_engine_child() { + let args = |task| { + engine_serve_http_args( + "lemonade", + "svc", + "Whisper-Tiny", + "127.0.0.1", + 11435, + "gpu_required", + &[], + None, + None, + None, + Path::new("state.json"), + task, + ) + }; + let transcription = args(rocm_core::ModelTask::Transcription); + let at = transcription + .iter() + .position(|arg| arg == "--task") + .expect("the task must reach the child"); + assert_eq!(transcription[at + 1], "transcription"); + assert!( + !args(rocm_core::ModelTask::Chat).contains(&"--task".to_owned()), + "a chat restart keeps the argv it always had" + ); + } } diff --git a/crates/rocm-core/src/config.rs b/crates/rocm-core/src/config.rs index 860ddcf0e7..1c5cfa1371 100644 --- a/crates/rocm-core/src/config.rs +++ b/crates/rocm-core/src/config.rs @@ -69,6 +69,66 @@ pub struct AutomationsConfig { pub watchers: BTreeMap, } +/// Port the background daemon serves API docs on when `config.json` sets none. +/// Kept clear of `DEFAULT_LOCAL_PORT` and the ports above it that consecutive +/// `rocm serve` launches pick. +pub const DEFAULT_API_DOCS_PORT: u16 = 11_430; + +/// API docs for running model servers, served by the background daemon on +/// loopback. Described in: README.md's API docs section, docs/testing.md and +/// docs/manual-testing.md. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ApiDocsConfig { + #[serde(default = "default_api_docs_enabled")] + pub enabled: bool, + #[serde(default = "default_api_docs_port")] + pub port: u16, +} + +impl Default for ApiDocsConfig { + fn default() -> Self { + Self { + enabled: default_api_docs_enabled(), + port: default_api_docs_port(), + } + } +} + +const fn default_api_docs_enabled() -> bool { + true +} + +const fn default_api_docs_port() -> u16 { + DEFAULT_API_DOCS_PORT +} + +impl ApiDocsConfig { + /// The port the docs are served on, or `None` when they are off: turned off + /// in `config.json`, or set to port 0, which would bind a random port no + /// link could name. + #[must_use] + pub const fn served_port(&self) -> Option { + if self.enabled && self.port != 0 { + Some(self.port) + } else { + None + } + } + + /// The docs server's root: `GET` lists every running server. + #[must_use] + pub fn base_url(&self) -> String { + crate::format_http_base_url(crate::DEFAULT_LOCAL_HOST, self.port) + } + + /// Swagger UI for one running server. The trailing slash is part of the + /// address: the page loads its assets relative to it. + #[must_use] + pub fn service_docs_url(&self, service_id: &str) -> String { + format!("{}/services/{service_id}/docs/", self.base_url()) + } +} + #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ProviderUserConfig { #[serde(default)] @@ -210,6 +270,8 @@ pub struct RocmCliConfig { pub engines: BTreeMap, #[serde(default)] pub automations: AutomationsConfig, + #[serde(default)] + pub api_docs: ApiDocsConfig, /// rocm-dash telemetry/dashboard knobs. Nested as a sub-config /// so it never collides with the rocm-cli `telemetry` analytics policy on /// rebase. Every field defaults, so the section is fully optional. diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index ec4163dc25..b89e632143 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -27,6 +27,7 @@ pub mod log_tail; pub mod managed_runtime; pub mod model_catalog; pub mod model_readiness; +pub mod model_task; pub mod net; pub mod openmpi; pub mod os_release; @@ -56,10 +57,11 @@ pub use automation::{ }; pub use binary_paths::{daemon_binary_path, engine_binary_path, sibling_binary_path}; pub use config::{ - AutomationsConfig, DashboardConfig, DashboardDaemonConfig, DashboardTuiConfig, - EngineUserConfig, ManagedToolConfig, PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, - PermissionsConfig, ProviderUserConfig, RocmCliConfig, SetupConfig, TELEMETRY_MODE_LOCAL, - TELEMETRY_MODE_OFF, TelemetryConfig, WatcherMode, WatcherUserConfig, + ApiDocsConfig, AutomationsConfig, DEFAULT_API_DOCS_PORT, DashboardConfig, + DashboardDaemonConfig, DashboardTuiConfig, EngineUserConfig, ManagedToolConfig, + PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, PermissionsConfig, ProviderUserConfig, + RocmCliConfig, SetupConfig, TELEMETRY_MODE_LOCAL, TELEMETRY_MODE_OFF, TelemetryConfig, + WatcherMode, WatcherUserConfig, }; pub use crypto::{ generate_rsa_signing_keypair, sign_rsa_pkcs1_sha256_signature, @@ -122,18 +124,20 @@ pub use model_catalog::{ normalize_model_ref, platform_matches_gfx_family, resolve_builtin_model_recipe, resolve_model_recipe, resolve_model_recipe_artifact, verify_model_recipe_index_signature, }; +pub use model_task::ModelTask; pub use net::{ Backoff, DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, DownloadRequest, EndpointReadiness, EndpointReadinessOutcome, HttpResponseParts, INFERENCE_PROBE_ATTEMPTED_STATE_KEY, INFERENCE_PROBE_RETRY_INTERVAL, INFERENCE_PROBE_TIMEOUT, - INFERENCE_VERIFIED_STATE_KEY, connect_tcp_stream, download_file_streaming, + INFERENCE_VERIFIED_STATE_KEY, MultipartFile, connect_tcp_stream, download_file_streaming, download_file_streaming_with_progress, download_file_to_path, download_file_to_path_with_progress, engine_state_inference_verified, format_host_for_url, format_host_port, format_http_base_url, http_get_text, http_get_text_with_auth, - http_post_json_with_auth, managed_service_endpoint_model_ready, + http_post_json_with_auth, http_post_multipart_with_auth, managed_service_endpoint_model_ready, managed_service_endpoint_readiness, merge_json_state_file, openai_chat_completion_probe, - openai_chat_completion_status, openai_models_endpoint_has_model, parse_http_endpoint, - read_http_response_bounded, write_all_tcp_stream, + openai_chat_completion_status, openai_inference_probe, openai_inference_status, + openai_models_endpoint_has_model, openai_transcription_status, parse_http_endpoint, + read_http_response_bounded, silent_wav_16khz_mono, write_all_tcp_stream, }; pub use paths::{AppPaths, engine_plugin_dirs}; pub use proc_lifecycle::{ diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index f970bbbbfb..53bc23b8a5 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -508,6 +508,59 @@ "warnings": [ "multimodal model; text chat is the default serve path" ] + }, + { + "canonical_model_id": "Whisper-Large-v3-Turbo", + "aliases": [ + "whisper", + "whisper-turbo" + ], + "task": "transcription", + "source": "recipe_index", + "revision": "main", + "loader": "whispercpp", + "trust_remote_code": false, + "dtype": "ggml", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 2, + "recommended_system_ram_gb": 8, + "quantization": "ggml f16; Lemonade whisper.cpp ROCm or Vulkan backend", + "artifact_hint": "Lemonade model id resolved and downloaded by the Lemonade engine", + "artifacts": [], + "engine_recipes": [], + "manual_alternatives": [], + "chat_template_mode": "none", + "preferred_engines": [ + "lemonade" + ], + "warnings": [ + "speech-to-text; serves POST /v1/audio/transcriptions, not chat" + ] + }, + { + "canonical_model_id": "Whisper-Tiny", + "aliases": [], + "task": "transcription", + "source": "recipe_index", + "revision": "main", + "loader": "whispercpp", + "trust_remote_code": false, + "dtype": "ggml", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 1, + "recommended_system_ram_gb": 4, + "quantization": "ggml f16; Lemonade whisper.cpp ROCm or Vulkan backend", + "artifact_hint": "Lemonade model id resolved and downloaded by the Lemonade engine", + "artifacts": [], + "engine_recipes": [], + "manual_alternatives": [], + "chat_template_mode": "none", + "preferred_engines": [ + "lemonade" + ], + "warnings": [ + "speech-to-text; smallest Whisper model, for quick checks rather than accuracy" + ] } ] } diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index 5bd43f3ebe..5ab2470de6 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -234,6 +234,13 @@ pub struct ModelRecipeRecord { } impl ModelRecipeRecord { + /// The recipe's task, parsed. [`ModelRecipeIndexDocument::validate`] rejects + /// a recipe whose task does not parse, so this only errors for a record + /// that bypassed validation. + pub fn model_task(&self) -> Result { + crate::ModelTask::parse(&self.task) + } + pub fn matches_ref(&self, model_ref: &str) -> bool { let normalized = normalize_model_ref(model_ref); normalize_model_ref(&self.canonical_model_id) == normalized @@ -297,6 +304,9 @@ impl ModelRecipeIndexDocument { let mut refs = BTreeMap::::new(); for recipe in &self.recipes { require_nonempty(&recipe.canonical_model_id, "canonical_model_id")?; + // A task this build does not know is not checked here: an index + // written for a newer rocm-cli must stay loadable, and such a recipe + // fails on its own when it is served (`ModelRecipeRecord::model_task`). require_nonempty(&recipe.task, "task")?; require_nonempty(&recipe.source, "source")?; require_nonempty(&recipe.revision, "revision")?; @@ -1060,6 +1070,33 @@ mod tests { assert!(error.contains("must not be empty")); } + #[test] + fn a_recipe_with_a_task_this_build_cannot_serve_fails_alone() { + let mut unservable = sample_recipe_with_artifact("Qwen/Test-1B", &["test-qwen"]); + unservable.task = "embedding".to_owned(); + let chat = sample_recipe_with_artifact("Qwen/Test-2B", &["test-qwen-2"]); + + ModelRecipeIndexDocument { + schema_version: 1, + source: Some("fixture".to_owned()), + generated_at_unix_ms: Some(123), + platforms: Vec::new(), + recipes: vec![unservable.clone(), chat.clone()], + } + .validate() + .expect("an index written for a newer rocm-cli stays loadable"); + + let error = unservable + .model_task() + .expect_err("serving the unknown task must fail") + .to_string(); + assert!( + error.contains("unsupported model task `embedding`"), + "{error}" + ); + assert_eq!(chat.model_task().expect("chat"), crate::ModelTask::Chat); + } + #[test] fn model_recipe_index_rejects_duplicate_engine_recipes() { let mut recipe = sample_recipe_with_artifact("Qwen/Test-1B", &["test-qwen"]); diff --git a/crates/rocm-core/src/model_task.rs b/crates/rocm-core/src/model_task.rs new file mode 100644 index 0000000000..a0f8d4022f --- /dev/null +++ b/crates/rocm-core/src/model_task.rs @@ -0,0 +1,158 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What a served model does: answer chat requests or transcribe speech. +//! +//! The task decides which OpenAI-compatible route a server exposes, how its +//! readiness is probed, and which commands may use it — `rocm chat` and the +//! dashboard chat pane need a chat model, and a speech-to-text server cannot +//! stand in for one. +//! +//! Records written before the task existed carry none, and every model served +//! then was a chat model, so a missing task deserializes as [`ModelTask::Chat`]. + +use anyhow::{Result, bail}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum ModelTask { + /// Chat completions: `POST /v1/chat/completions`. + #[default] + Chat, + /// Speech-to-text: `POST /v1/audio/transcriptions`. + Transcription, + /// A task written by a newer rocm-cli or engine plugin that this build does + /// not know. Kept so that one unfamiliar value degrades only the record or + /// capability it sits in, instead of failing the whole message: it is not + /// chat, and it is never served or probed. + #[serde(other)] + Unknown, +} + +impl ModelTask { + /// The value written to catalog recipes and service records. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Chat => "chat", + Self::Transcription => "transcription", + Self::Unknown => "unknown", + } + } + + /// Plain-English name for user-facing output. + #[must_use] + pub const fn display_name(self) -> &'static str { + match self { + Self::Chat => "chat", + Self::Transcription => "speech-to-text", + Self::Unknown => "unrecognised", + } + } + + /// The route a client calls, relative to the server's `/v1` base URL. + /// Empty for [`ModelTask::Unknown`], whose route this build cannot name. + #[must_use] + pub const fn api_route(self) -> &'static str { + match self { + Self::Chat => "/chat/completions", + Self::Transcription => "/audio/transcriptions", + Self::Unknown => "", + } + } + + #[must_use] + pub const fn is_chat(self) -> bool { + matches!(self, Self::Chat) + } + + /// Refuse to serve, restart or recover a server for a task this build does + /// not know. Such a task was recorded by a newer rocm-cli; serving it as + /// anything this build knows would serve the wrong thing. + pub fn ensure_servable(self) -> Result<()> { + if matches!(self, Self::Unknown) { + bail!( + "this rocm-cli does not know the model task recorded for this server; a newer \ + rocm-cli recorded it and is needed to serve it" + ); + } + Ok(()) + } + + /// Parse a recipe's `task` string. + /// + /// Unknown values are an error rather than a silent `Chat`: a recipe that + /// names a task this build cannot serve must not be launched and probed as a + /// chat model. + pub fn parse(value: &str) -> Result { + match value.trim().to_ascii_lowercase().as_str() { + "chat" => Ok(Self::Chat), + "transcription" => Ok(Self::Transcription), + other => bail!("unsupported model task `{other}`; expected `chat` or `transcription`"), + } + } +} + +impl std::fmt::Display for ModelTask { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn missing_task_deserializes_as_chat_for_older_records() { + #[derive(Deserialize)] + struct Record { + #[serde(default)] + task: ModelTask, + } + let record: Record = serde_json::from_str("{}").expect("parse"); + assert_eq!(record.task, ModelTask::Chat); + } + + #[test] + fn task_round_trips_through_its_wire_name() { + for task in [ModelTask::Chat, ModelTask::Transcription] { + let json = serde_json::to_string(&task).expect("serialize"); + assert_eq!(json, format!("\"{}\"", task.as_str())); + assert_eq!(ModelTask::parse(task.as_str()).expect("parse"), task); + } + } + + #[test] + fn unknown_task_is_rejected_rather_than_treated_as_chat() { + let error = ModelTask::parse("embedding").expect_err("embedding is not servable"); + assert!( + error + .to_string() + .contains("unsupported model task `embedding`") + ); + } + + #[test] + fn an_unrecognised_task_from_a_newer_writer_degrades_only_its_own_entry() { + #[derive(Deserialize)] + struct Capabilities { + tasks: Vec, + } + let parsed: Capabilities = + serde_json::from_str(r#"{"tasks":["chat","embedding"]}"#).expect("parse"); + assert_eq!(parsed.tasks, vec![ModelTask::Chat, ModelTask::Unknown]); + assert!(!ModelTask::Unknown.is_chat()); + } + + #[test] + fn each_task_names_its_own_route() { + assert_eq!(ModelTask::Chat.api_route(), "/chat/completions"); + assert_eq!( + ModelTask::Transcription.api_route(), + "/audio/transcriptions" + ); + } +} diff --git a/crates/rocm-core/src/net.rs b/crates/rocm-core/src/net.rs index 03b6616cb6..fce0b9069c 100644 --- a/crates/rocm-core/src/net.rs +++ b/crates/rocm-core/src/net.rs @@ -559,6 +559,72 @@ pub fn http_post_json_with_auth( body: &serde_json::Value, endpoint_api_key: Option<&str>, timeout: Duration, +) -> Result { + let payload = serde_json::to_vec(body).context("failed to serialize HTTP JSON body")?; + http_post_bytes_with_auth( + endpoint_url, + path, + "application/json", + &payload, + endpoint_api_key, + timeout, + ) +} + +/// POST a `multipart/form-data` body of text `fields` plus one file part. +pub fn http_post_multipart_with_auth( + endpoint_url: &str, + path: &str, + fields: &[(&str, &str)], + file: MultipartFile<'_>, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + const BOUNDARY: &str = "rocm-cli-multipart-boundary-7d3f1c"; + let mut payload = Vec::new(); + for (name, value) in fields { + payload.extend_from_slice( + format!( + "--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n" + ) + .as_bytes(), + ); + } + payload.extend_from_slice( + format!( + "--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"{}\"; filename=\"{}\"\r\nContent-Type: {}\r\n\r\n", + file.field, file.filename, file.content_type + ) + .as_bytes(), + ); + payload.extend_from_slice(file.bytes); + payload.extend_from_slice(format!("\r\n--{BOUNDARY}--\r\n").as_bytes()); + http_post_bytes_with_auth( + endpoint_url, + path, + &format!("multipart/form-data; boundary={BOUNDARY}"), + &payload, + endpoint_api_key, + timeout, + ) +} + +/// The file part of a [`http_post_multipart_with_auth`] request. +#[derive(Debug, Clone, Copy)] +pub struct MultipartFile<'a> { + pub field: &'a str, + pub filename: &'a str, + pub content_type: &'a str, + pub bytes: &'a [u8], +} + +fn http_post_bytes_with_auth( + endpoint_url: &str, + path: &str, + content_type: &str, + payload: &[u8], + endpoint_api_key: Option<&str>, + timeout: Duration, ) -> Result { let deadline = Instant::now() + timeout; let (host, port) = parse_http_endpoint(endpoint_url) @@ -569,12 +635,13 @@ pub fn http_post_json_with_auth( Some(key) => format!("Authorization: Bearer {key}\r\n"), None => String::new(), }; - let payload = serde_json::to_string(body).context("failed to serialize HTTP JSON body")?; - let request = format!( - "POST {path} HTTP/1.1\r\nHost: {host_header}\r\nContent-Type: application/json\r\nAccept: application/json\r\nContent-Length: {}\r\n{auth_header}Connection: close\r\n\r\n{payload}", + let head = format!( + "POST {path} HTTP/1.1\r\nHost: {host_header}\r\nContent-Type: {content_type}\r\nAccept: application/json\r\nContent-Length: {}\r\n{auth_header}Connection: close\r\n\r\n", payload.len() ); - write_all_tcp_stream(&mut stream, request.as_bytes()) + let mut request = head.into_bytes(); + request.extend_from_slice(payload); + write_all_tcp_stream(&mut stream, &request) .with_context(|| format!("failed to write HTTP POST {path}"))?; let response = read_http_response_bounded(&mut stream, deadline) .with_context(|| format!("failed to read HTTP POST {path}"))?; @@ -659,6 +726,7 @@ pub fn merge_json_state_file(path: &Path, patch: &serde_json::Value) -> Result<( /// still leaves the throttle in place instead of freeing the next poll to spend /// another full timeout. pub fn engine_state_inference_verified( + task: crate::ModelTask, state_path: &Path, state: Option<&serde_json::Value>, endpoint_url: &str, @@ -686,7 +754,8 @@ pub fn engine_state_inference_verified( state_path, &serde_json::json!({ INFERENCE_PROBE_ATTEMPTED_STATE_KEY: now }), ); - if !openai_chat_completion_probe( + if !openai_inference_probe( + task, endpoint_url, model_ref, endpoint_api_key, @@ -728,10 +797,10 @@ fn http_status_code(status_line: &str) -> Option { /// non-empty content would also wrongly fail a reasoning model, which can spend /// its whole (tiny) token budget before emitting any content. /// -/// The rule does mean a `404` reads as serving. That is harmless for the engines -/// shipped today — both implement `/v1/chat/completions`, and a wrong key fails -/// the model listing that gates this call — but an engine that does not expose an -/// OpenAI-shaped chat route would need a different signal rather than this one. +/// The rule does mean a `404` (or Lemonade's `400` for a model that is not an +/// LLM) reads as serving, so this is only a readiness signal for a chat model. +/// A service of another task is probed with its own request through +/// [`openai_inference_probe`], which is what every readiness check calls. pub fn openai_chat_completion_probe( endpoint_url: &str, model_ref: &str, @@ -771,6 +840,107 @@ pub fn openai_chat_completion_status( Ok(response.status) } +/// Ask a speech-to-text endpoint to transcribe one second of silence and +/// return the HTTP status — the transcription counterpart of +/// [`openai_chat_completion_status`]. +/// +/// Silence keeps the probe cheap and deterministic: the request exercises the +/// whole decode path (multipart upload, audio decode, model forward pass) +/// without depending on what a model makes of real speech. +pub fn openai_transcription_status( + endpoint_url: &str, + model_ref: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + let wav = silent_wav_16khz_mono(Duration::from_secs(1)); + let response = http_post_multipart_with_auth( + endpoint_url, + "/v1/audio/transcriptions", + &[("model", model_ref)], + MultipartFile { + field: "file", + filename: "silence.wav", + content_type: "audio/wav", + bytes: &wav, + }, + endpoint_api_key, + timeout, + )?; + Ok(response.status) +} + +/// The smallest real request for a model's task, as an HTTP status. +pub fn openai_inference_status( + task: crate::ModelTask, + endpoint_url: &str, + model_ref: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + match task { + crate::ModelTask::Chat => { + openai_chat_completion_status(endpoint_url, model_ref, endpoint_api_key, timeout) + } + crate::ModelTask::Transcription => { + openai_transcription_status(endpoint_url, model_ref, endpoint_api_key, timeout) + } + // Readiness is never claimed for a service this build cannot probe. + crate::ModelTask::Unknown => bail!("no inference probe exists for an unrecognised task"), + } +} + +/// Whether the endpoint answers the request that matches the model's task. +/// +/// A chat server counts as serving on the same "answered below 500" rule as +/// [`openai_chat_completion_probe`]. A speech-to-text server must answer 200: +/// the probe sends a valid recording, so a 4xx means the route or the model is +/// not there (a model not loaded, a decode failure), not that it is serving. +/// Probing a speech-to-text server with a chat request would instead read its +/// `404` as serving. +pub fn openai_inference_probe( + task: crate::ModelTask, + endpoint_url: &str, + model_ref: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + let status = openai_inference_status(task, endpoint_url, model_ref, endpoint_api_key, timeout)?; + Ok(match task { + crate::ModelTask::Transcription => status == 200, + _ => status < 500, + }) +} + +/// A PCM WAV file of `duration` of silence, 16 kHz mono 16-bit — the input +/// format speech-to-text servers accept without resampling. +#[must_use] +pub fn silent_wav_16khz_mono(duration: Duration) -> Vec { + const SAMPLE_RATE: u32 = 16_000; + const BYTES_PER_SAMPLE: u16 = 2; + // Keeps the RIFF size field (`36 + data_len`) inside `u32`. + const MAX_SAMPLES: u32 = (u32::MAX - 44) / BYTES_PER_SAMPLE as u32; + let samples = u32::try_from(duration.as_millis() * u128::from(SAMPLE_RATE) / 1000) + .unwrap_or(MAX_SAMPLES) + .min(MAX_SAMPLES); + let data_len = samples * u32::from(BYTES_PER_SAMPLE); + let mut wav = Vec::with_capacity(44 + data_len as usize); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(36 + data_len).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16_u32.to_le_bytes()); // fmt chunk size + wav.extend_from_slice(&1_u16.to_le_bytes()); // PCM + wav.extend_from_slice(&1_u16.to_le_bytes()); // mono + wav.extend_from_slice(&SAMPLE_RATE.to_le_bytes()); + wav.extend_from_slice(&(SAMPLE_RATE * u32::from(BYTES_PER_SAMPLE)).to_le_bytes()); + wav.extend_from_slice(&BYTES_PER_SAMPLE.to_le_bytes()); // block align + wav.extend_from_slice(&16_u16.to_le_bytes()); // bits per sample + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_len.to_le_bytes()); + wav.resize(44 + data_len as usize, 0); + wav +} + pub fn openai_models_endpoint_has_model( endpoint_url: &str, expected_model: Option<&str>, @@ -886,7 +1056,8 @@ pub fn managed_service_endpoint_readiness( record.canonical_model_id.as_str() }; record.inference_probe_attempted_at_unix_ms = Some(now); - if !openai_chat_completion_probe( + if !openai_inference_probe( + record.task, &record.endpoint_url, model_ref, endpoint_api_key, @@ -2244,6 +2415,98 @@ mod tests { Ok(()) } #[test] + fn a_speech_to_text_service_is_probed_with_a_transcription_not_a_chat_request() -> Result<()> { + // A speech-to-text server has no chat route; a chat probe would read its + // 404 as "serving" without ever touching the model. The probe must send + // the request the service actually exists to answer. + let (port, server) = spawn_canned_http_server(vec![ + Some(("HTTP/1.1 200 OK", MODELS_OK_BODY)), + Some(("HTTP/1.1 200 OK", r#"{"text":""}"#)), + ])?; + let mut record = probe_test_record(port); + record.task = crate::ModelTask::Transcription; + + assert_eq!( + managed_service_endpoint_readiness( + &mut record, + None, + Duration::from_secs(2), + Duration::from_secs(2) + ) + .readiness, + EndpointReadiness::Serving + ); + + let requests = server.join().expect("server thread should not panic")?; + let probe = requests.get(1).expect("an inference probe was sent"); + assert!( + probe.starts_with("POST /v1/audio/transcriptions HTTP/1.1"), + "{probe}" + ); + assert!(probe.contains("multipart/form-data; boundary=")); + assert!(probe.contains("name=\"model\"\r\n\r\nQwen/Qwen3-0.6B\r\n")); + assert!(probe.contains("filename=\"silence.wav\"")); + assert!(probe.contains("RIFF"), "the uploaded file is a WAV"); + assert!( + requests + .iter() + .all(|request| !request.contains("/v1/chat/completions")), + "no chat request may be sent to a speech-to-text service" + ); + Ok(()) + } + #[test] + fn a_speech_to_text_service_answering_5xx_is_not_ready() -> Result<()> { + let (port, server) = spawn_canned_http_server(vec![ + Some(("HTTP/1.1 200 OK", MODELS_OK_BODY)), + Some(( + "HTTP/1.1 500 Internal Server Error", + r#"{"error":"backend failed to start"}"#, + )), + ])?; + let mut record = probe_test_record(port); + record.task = crate::ModelTask::Transcription; + + assert_eq!( + managed_service_endpoint_readiness( + &mut record, + None, + Duration::from_secs(2), + Duration::from_secs(2) + ) + .readiness, + EndpointReadiness::Listing + ); + assert!(record.inference_verified_at_unix_ms.is_none()); + let requests = server.join().expect("server thread should not panic")?; + assert!( + requests + .get(1) + .is_some_and(|probe| probe.starts_with("POST /v1/audio/transcriptions ")), + "the 500 must be the transcription probe's answer: {requests:?}" + ); + Ok(()) + } + #[test] + fn silent_wav_is_a_well_formed_16khz_mono_pcm_file() { + let wav = silent_wav_16khz_mono(Duration::from_secs(1)); + assert_eq!(wav.len(), 44 + 32_000); + assert_eq!(&wav[0..4], b"RIFF"); + assert_eq!( + u32::from_le_bytes(wav[4..8].try_into().unwrap()), + 36 + 32_000 + ); + assert_eq!(&wav[8..16], b"WAVEfmt "); + assert_eq!( + u16::from_le_bytes(wav[22..24].try_into().unwrap()), + 1, + "mono" + ); + assert_eq!(u32::from_le_bytes(wav[24..28].try_into().unwrap()), 16_000); + assert_eq!(&wav[36..40], b"data"); + assert!(wav[44..].iter().all(|byte| *byte == 0), "silence"); + } + #[test] fn a_warming_service_is_not_re_probed_on_every_poll() -> Result<()> { // Only a successful probe latches, so a model that is listed but still // loading would otherwise be re-probed by every poll — and each attempt @@ -2330,6 +2593,30 @@ mod tests { Ok(()) } #[test] + fn a_speech_to_text_probe_needs_a_transcript_not_just_an_answer() -> Result<()> { + for (status_line, serving) in [ + ("HTTP/1.1 200 OK", true), + ("HTTP/1.1 404 Not Found", false), + ("HTTP/1.1 400 Bad Request", false), + ] { + let (port, server) = + spawn_canned_http_server(vec![Some((status_line, r#"{"text":""}"#))])?; + assert_eq!( + openai_inference_probe( + crate::ModelTask::Transcription, + &format!("http://127.0.0.1:{port}/v1"), + "Whisper-Tiny", + None, + Duration::from_secs(2), + )?, + serving, + "{status_line}" + ); + server.join().expect("server thread should not panic")?; + } + Ok(()) + } + #[test] fn inference_probe_sends_the_service_key_to_a_protected_endpoint() -> Result<()> { let (port, server) = spawn_canned_http_server(vec![ Some(("HTTP/1.1 200 OK", MODELS_OK_BODY)), diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index ce04590010..93f2966933 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -31,6 +31,11 @@ pub struct ManagedServiceRecord { pub engine: String, pub model_ref: String, pub canonical_model_id: String, + /// What the served model does, which decides its client route and whether + /// chat commands may use it. Records written before this field existed all + /// served chat models, which is what the default restores. + #[serde(default)] + pub task: crate::ModelTask, pub host: String, pub port: u16, pub endpoint_url: String, @@ -134,6 +139,7 @@ impl ManagedServiceRecord { engine, model_ref: model_ref.into(), canonical_model_id: canonical_model_id.into(), + task: crate::ModelTask::Chat, host, port, mode: mode.into(), @@ -620,4 +626,29 @@ mod tests { } Ok(()) } + + #[test] + fn a_record_written_before_tasks_existed_is_a_chat_service() { + let (_root, paths) = temp_app_paths("record-without-task"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-old", + "lemonade", + "qwen", + "Qwen3-4B-Instruct-2507-GGUF", + "127.0.0.1", + 11435, + "managed", + 1, + None, + None, + None, + ); + record.task = crate::ModelTask::Transcription; + let mut json = serde_json::to_value(&record).expect("serialize"); + json.as_object_mut().expect("object").remove("task"); + + let older: ManagedServiceRecord = serde_json::from_value(json).expect("deserialize"); + assert_eq!(older.task, crate::ModelTask::Chat); + } } diff --git a/crates/rocm-dash-tui/src/app/chat.rs b/crates/rocm-dash-tui/src/app/chat.rs index fc0224b075..ae951b62b9 100644 --- a/crates/rocm-dash-tui/src/app/chat.rs +++ b/crates/rocm-dash-tui/src/app/chat.rs @@ -170,7 +170,7 @@ pub(crate) fn pick_managed_chat_endpoint( .get("status") .and_then(serde_json::Value::as_str) .is_some_and(|s| s.eq_ignore_ascii_case("ready")); - if !ready { + if !ready || !record_serves_chat(record) { continue; } let endpoint_present = record @@ -211,18 +211,56 @@ pub(crate) fn pick_managed_chat_endpoint( Some(ManagedChatEndpoint { base_url, model }) } -/// Query the bin's read-only `services` tool and, if a ready OpenAI-compatible -/// managed service exists, return its endpoint as an [`LlmConfig`]. +/// Whether a `services` record serves chat. Records written before services +/// carried a task were all chat services, so a missing task counts as chat. +fn record_serves_chat(record: &serde_json::Value) -> bool { + record + .get("task") + .and_then(serde_json::Value::as_str) + .is_none_or(|task| task.eq_ignore_ascii_case("chat")) +} + +/// Whether a live managed service that does not serve chat — a speech-to-text +/// server, say — is bound to `base_url`'s port. /// -/// This is how the dash learns the *actual* port it launched an engine on (e.g. -/// a tool-launched vLLM on a non-default port) instead of guessing the -/// well-known defaults. The seam call is blocking, so it runs off the reactor. -/// A best-effort `/v1/models` fetch both confirms the endpoint is live and -/// supplies the served model id; on fetch failure the registry-reported model -/// is used (the service was already readiness-verified by the bin). -pub(super) async fn detect_managed_chat( +/// The well-known-port fallback finds *a* server and adopts the first model it +/// lists; on a port held by a speech-to-text server that would set the chat +/// pane up to send chat requests to Whisper. The registry knows better. +pub(crate) fn non_chat_service_owns(services_result: &serde_json::Value, base_url: &str) -> bool { + let port_of = |url: &str| reqwest::Url::parse(url).ok()?.port_or_known_default(); + let Some(port) = port_of(base_url) else { + return false; + }; + services_result + .get("structuredContent") + .and_then(|s| s.get("services")) + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter(|record| !record_serves_chat(record)) + .filter(|record| { + record + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|status| { + ["ready", "running", "starting", "recovering"] + .iter() + .any(|live| status.eq_ignore_ascii_case(live)) + }) + }) + .filter_map(|record| { + record + .get("endpoint_url") + .and_then(serde_json::Value::as_str) + }) + .any(|endpoint| port_of(endpoint) == Some(port)) +} + +/// The bin's read-only `services` tool payload. The seam call is blocking, so +/// it runs off the reactor. +async fn fetch_services( executor: Option, -) -> Option { +) -> Option { let executor = executor?; let outcome = tokio::task::spawn_blocking(move || executor.execute("services", &serde_json::json!({}))) @@ -231,7 +269,20 @@ pub(super) async fn detect_managed_chat( let crate::tool_exec::RocmToolOutcome::Result(value) = outcome else { return None; }; - let picked = pick_managed_chat_endpoint(&value)?; + Some(value) +} + +/// Query the bin's read-only `services` tool and, if a ready OpenAI-compatible +/// managed service exists, return its endpoint as an [`LlmConfig`]. +/// +/// This is how the dash learns the *actual* port it launched an engine on (e.g. +/// a tool-launched vLLM on a non-default port) instead of guessing the +/// well-known defaults. The seam call is blocking, so it runs off the reactor. +/// A best-effort `/v1/models` fetch both confirms the endpoint is live and +/// supplies the served model id; on fetch failure the registry-reported model +/// is used (the service was already readiness-verified by the bin). +async fn detect_managed_chat(services: &serde_json::Value) -> Option { + let picked = pick_managed_chat_endpoint(services)?; match fetch_first_model(&picked.base_url).await { Some(model) => Some(crate::llm::detected_llm_config(&picked.base_url, &model)), None => picked @@ -260,12 +311,21 @@ async fn detect_local_chat_with_probe( executor: Option, probe: impl Fn() -> Option<&'static str> + Send + 'static, ) -> Option { - if let Some(cfg) = detect_managed_chat(executor).await { + let services = fetch_services(executor).await; + if let Some(services) = services.as_ref() + && let Some(cfg) = detect_managed_chat(services).await + { return Some(cfg); } // TCP probe is blocking; keep it off the async reactor. let base = tokio::task::spawn_blocking(probe).await.ok().flatten()?; + if services + .as_ref() + .is_some_and(|services| non_chat_service_owns(services, base)) + { + return None; + } // Best-effort model query; fall back to the neutral default on any failure. let model = fetch_first_model(base) @@ -517,6 +577,74 @@ mod tests { ); } + #[test] + fn skips_a_ready_speech_to_text_service() { + // The newest ready Lemonade service is a speech-to-text server; the chat + // pane must keep the older chat service instead of routing chat to it. + let mut whisper = service( + "lemonade", + "http://127.0.0.1:13306/v1", + "ready", + "Whisper-Large-v3-Turbo", + 200, + ); + whisper["task"] = serde_json::json!("transcription"); + let env = services_envelope(serde_json::json!([ + service( + "lemonade", + "http://127.0.0.1:13305/v1", + "ready", + "qwen", + 100 + ), + whisper, + ])); + let picked = pick_managed_chat_endpoint(&env).expect("the chat service"); + assert_eq!(picked.base_url, "http://127.0.0.1:13305/v1"); + } + + #[test] + fn a_record_without_a_task_is_a_chat_service() { + let env = services_envelope(serde_json::json!([service( + "lemonade", + "http://127.0.0.1:13305/v1", + "ready", + "qwen", + 100 + )])); + assert!(pick_managed_chat_endpoint(&env).is_some()); + } + + #[test] + fn a_well_known_port_held_by_a_speech_to_text_service_is_not_a_chat_endpoint() { + let mut whisper = service( + "lemonade", + "http://127.0.0.1:13305/v1", + "ready", + "Whisper-Tiny", + 100, + ); + whisper["task"] = serde_json::json!("transcription"); + let env = services_envelope(serde_json::json!([whisper.clone()])); + assert!(non_chat_service_owns(&env, "http://localhost:13305/v1")); + assert!(!non_chat_service_owns(&env, "http://127.0.0.1:8000/v1")); + + whisper["status"] = serde_json::json!("stopped"); + let stopped = services_envelope(serde_json::json!([whisper])); + assert!( + !non_chat_service_owns(&stopped, "http://127.0.0.1:13305/v1"), + "a stopped speech service no longer holds the port" + ); + let chat = services_envelope(serde_json::json!([service( + "lemonade", + "http://127.0.0.1:13305/v1", + "ready", + "qwen", + 100 + )])); + assert!(!non_chat_service_owns(&chat, "http://127.0.0.1:13305/v1")); + } + #[test] fn skips_ready_record_with_empty_endpoint() { let env = services_envelope(serde_json::json!([service("vllm", "", "ready", "m", 100)])); @@ -638,6 +766,50 @@ mod tests { assert_eq!(cfg.base_url, PROBED); } + /// An executor whose read-only `services` tool reports `services`. + #[derive(Debug)] + struct ServicesOnly(serde_json::Value); + + impl crate::tool_exec::RocmToolExecutor for ServicesOnly { + fn execute( + &self, + _name: &str, + _args: &serde_json::Value, + ) -> crate::tool_exec::RocmToolOutcome { + crate::tool_exec::RocmToolOutcome::Result(serde_json::json!({ + "structuredContent": { "services": self.0.clone() } + })) + } + + fn execute_approved( + &self, + name: &str, + args: &serde_json::Value, + ) -> crate::tool_exec::RocmToolOutcome { + self.execute(name, args) + } + } + + #[tokio::test] + async fn the_port_fallback_does_not_adopt_a_speech_to_text_server() { + // The well-known-port probe finds a server on the port a running + // speech-to-text service is recorded on: the chat pane must not be set + // up to send chat requests to it. + const PROBED: &str = "http://127.0.0.1:1/v1"; + let executor: crate::tool_exec::SharedRocmToolExecutor = + std::sync::Arc::new(ServicesOnly(serde_json::json!([{ + "service_id": "svc-whisper", + "task": "transcription", + "status": "ready", + "endpoint_url": PROBED, + }]))); + assert!( + detect_local_chat_with_probe(Some(executor), || Some(PROBED)) + .await + .is_none() + ); + } + #[test] fn should_detect_only_when_no_url_env_or_key_configured() { // The clean slate: nothing configured → auto-detect a local engine. diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index fbca3104fc..f18b7e2e5c 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -9,6 +9,8 @@ use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; +pub use rocm_core::ModelTask; + pub const ENGINE_PROTOCOL_VERSION: &str = "0.1.0"; pub const ENGINE_RECIPE_CONTRACT_VERSION: &str = "0.1.0"; pub const ENGINE_PLUGIN_BINARY_PREFIX: &str = "rocm-engine-"; @@ -391,6 +393,10 @@ pub struct ResolveModelRequest { pub recipe_override: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub engine_recipe: Option, + /// What the model does. Omitted (and defaulted) for chat, so a request from + /// an older CLI and one for a chat model look the same to an engine. + #[serde(default, skip_serializing_if = "is_chat_task")] + pub task: ModelTask, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -407,6 +413,14 @@ pub struct LaunchRequest { pub engine_recipe: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub gpu_selection: Option, + /// What the model does; see [`ResolveModelRequest::task`]. + #[serde(default, skip_serializing_if = "is_chat_task")] + pub task: ModelTask, +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // serde's skip_serializing_if passes a reference +const fn is_chat_task(task: &ModelTask) -> bool { + task.is_chat() } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -448,6 +462,22 @@ pub struct EngineCapabilities { pub tool_calling: bool, pub quantized_models: String, pub reasoning_parser: bool, + /// The model tasks this engine can serve. An engine that predates this + /// field served chat models only, which is what the default restores, so + /// `rocm serve` refuses to hand it any other kind of model. + #[serde(default = "default_engine_tasks")] + pub tasks: Vec, +} + +fn default_engine_tasks() -> Vec { + vec![ModelTask::Chat] +} + +impl EngineCapabilities { + #[must_use] + pub fn serves(&self, task: ModelTask) -> bool { + self.tasks.contains(&task) + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -600,6 +630,44 @@ impl HealthcheckResponse { } } +/// The OpenAI-compatible routes every chat server answers. +/// +/// Each engine reports them, after its own health route, in +/// [`EndpointResponse::supported_routes`]; the daemon's API docs are tested +/// against [`supported_routes`] for every task, so the lists agree. +pub const OPENAI_CHAT_ROUTES: [&str; 3] = ["/v1/models", "/v1/chat/completions", "/v1/completions"]; + +/// The OpenAI-compatible routes every speech-to-text server answers. +pub const OPENAI_TRANSCRIPTION_ROUTES: [&str; 2] = ["/v1/models", "/v1/audio/transcriptions"]; + +/// The OpenAI-compatible routes a server for `task` answers, besides its +/// health route. +#[must_use] +pub const fn task_routes(task: ModelTask) -> &'static [&'static str] { + match task { + ModelTask::Chat => &OPENAI_CHAT_ROUTES, + ModelTask::Transcription => &OPENAI_TRANSCRIPTION_ROUTES, + // A task this build does not know: only the route every server has. + ModelTask::Unknown => &["/v1/models"], + } +} + +/// `supported_routes` for a server for `task` whose health route is +/// `health_route`. +#[must_use] +pub fn supported_routes(health_route: &str, task: ModelTask) -> Vec { + std::iter::once(health_route) + .chain(task_routes(task).iter().copied()) + .map(str::to_owned) + .collect() +} + +/// `supported_routes` for a chat server whose health route is `health_route`. +#[must_use] +pub fn chat_supported_routes(health_route: &str) -> Vec { + supported_routes(health_route, ModelTask::Chat) +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EndpointResponse { pub endpoint_url: String, @@ -898,6 +966,7 @@ mod tests { }], notes: vec!["signed recipe metadata".to_owned()], }), + task: ModelTask::Chat, }; let serialized = serde_json::to_string(&request).unwrap(); @@ -944,6 +1013,7 @@ mod tests { notes: Vec::new(), }), gpu_selection: None, + task: ModelTask::Chat, }; let serialized = serde_json::to_string(&request).unwrap(); @@ -1054,4 +1124,70 @@ mod tests { std::process::id() )) } + + #[test] + fn each_task_reports_its_routes_after_the_health_route() { + assert_eq!( + chat_supported_routes("/health"), + [ + "/health", + "/v1/models", + "/v1/chat/completions", + "/v1/completions" + ] + ); + assert_eq!( + supported_routes("/v1/health", ModelTask::Transcription), + ["/v1/health", "/v1/models", "/v1/audio/transcriptions"] + ); + // A task this build cannot name: only the routes every server has. + assert_eq!( + supported_routes("/v1/health", ModelTask::Unknown), + ["/v1/health", "/v1/models"] + ); + } + + #[test] + fn capabilities_from_an_engine_without_tasks_advertise_chat_only() { + let older = serde_json::json!({ + "cpu": false, + "rocm_gpu": true, + "openai_compatible": true, + "tool_calling": false, + "quantized_models": "none", + "reasoning_parser": false + }); + let capabilities: EngineCapabilities = serde_json::from_value(older).unwrap(); + assert_eq!(capabilities.tasks, vec![ModelTask::Chat]); + assert!(capabilities.serves(ModelTask::Chat)); + assert!(!capabilities.serves(ModelTask::Transcription)); + } + + #[test] + fn a_chat_launch_request_serializes_without_a_task_field() { + let request = LaunchRequest { + service_id: "svc".to_owned(), + env_id: None, + runtime_id: None, + model_ref: "Qwen/Test".to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: Some(DevicePolicy::GpuRequired), + endpoint_mode: None, + engine_recipe: None, + gpu_selection: None, + task: ModelTask::Chat, + }; + let chat = serde_json::to_value(&request).unwrap(); + assert!(chat.get("task").is_none(), "{chat}"); + + let transcription = serde_json::to_value(LaunchRequest { + task: ModelTask::Transcription, + ..request + }) + .unwrap(); + assert_eq!(transcription["task"], "transcription"); + let parsed: LaunchRequest = serde_json::from_value(transcription).unwrap(); + assert_eq!(parsed.task, ModelTask::Transcription); + } } diff --git a/docs/architecture.md b/docs/architecture.md index d2e1c90348..84f931b220 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -29,6 +29,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every [`main.rs`](../apps/rocm/src/main.rs) itself is **partially modularized** — Phase 6a (ROCMAI-82) extracted the driver-install/engines/serve clusters, Phase 6b (ROCMAI-91) extracted the managed-process-launch cluster, Phase 6c (ROCMAI-89) extracted the `runtimes` command/uninstall/rollback/SDK-install-finalization clusters, Phase 6d (ROCMAI-88) extracted the chat/assistant and MCP tool-bridge cluster below across three PRs; the rest of the split is planned across several more PRs, one cluster at a time (see EAI-7768). +- [`api_docs_link.rs`](../apps/rocm/src/api_docs_link.rs) — full domain extraction: the API docs link printed by `rocm serve` and `rocm services list`, probed against the docs server before it is printed. The docs themselves are served by `rocmd` ([`api_docs.rs`](../apps/rocmd/src/api_docs.rs)). - [`automations.rs`](../apps/rocm/src/automations.rs) — mechanical relocation, dispatch-adjacent, no owned types. - [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) — a further-extracted variant of full domain extraction: it owns its clap command enum (`BootstrapCommand`) and dispatch function too, rather than leaving them in [`main.rs`](../apps/rocm/src/main.rs). - [`chat.rs`](../apps/rocm/src/chat.rs) — full domain extraction (Phase 6d, ROCMAI-88), mirrors [`providers.rs`](../apps/rocm/src/providers.rs). Owns the chat/MCP-bridge cluster's types (`ChatRocmCommandAction`, `CommandCapture`, `DeterministicModelRecipe`, `DeterministicServiceRow`, `DeterministicEngineRow`, and others — all module-private except `ChatToolApprovalRequest`, reached by `dash_seam.rs` as `chat_tool_approval_request`'s return type), its validation/approval/execution logic (`validate_chat_tool_call` and the `validate_chat_*_tool_call` family), deterministic tool-output summarization (including `render_services_tool_result_text`), and the MCP tool-bridge entry points (`run_internal_mcp_call` and its helpers). [`main.rs`](../apps/rocm/src/main.rs), [`dash.rs`](../apps/rocm/src/dash.rs), and [`dash_seam.rs`](../apps/rocm/src/dash_seam.rs) call into it in production; [`comfyui.rs`](../apps/rocm/src/comfyui.rs) and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) reference it only from their own `#[cfg(test)]` modules (`deterministic_comfyui_tool_summary` and `rocm_chat_tool_requested_args` respectively) — every one of these five files' references still goes through the `chat::` module boundary. As a child module of the crate root (`mod chat;` in `main.rs`), Rust's module-privacy rule already makes most of `main.rs`'s private helpers visible to `chat.rs` without any visibility change, when `chat.rs` calls into `main.rs`. The other direction needs real widening: `chat.rs`'s own functions reached from those five files — `run_internal_mcp_call` (a production call from `main.rs`), `rocm_chat_tool_requested_args` (reached only by `runtime_install_finalize.rs`'s own test module), among others — stay `pub(crate)`; the third PR's visibility audit tightened every other function to module-private, leaving 20 of `chat.rs`'s 123 functions `pub(crate)`, each with a confirmed external caller, production or test-only. The cluster's regression tests live in `chat.rs`'s own `mod tests`, moved out of `main.rs`'s shared `mod tests` block across the second and third PRs; two `main.rs` tests that reference `chat::` only incidentally — structural checks on `main.rs`'s own `dispatch()` source (`command_chat_honors_chat_mock_and_keeps_prompt_passthrough`, `command_chat_reads_prompt_from_piped_stdin`) — plus `serve_negative_sampling_space_form_reaches_range_validator`, a shared clap value-parser wiring test living alongside its sibling command-parsing tests, stay in `main.rs`. @@ -59,6 +60,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every Modularized (ROCMAI-83, Phase 5 of EAI-7768's sequencing): +- [`api_docs.rs`](../apps/rocmd/src/api_docs.rs) — added after that split, as its own module from the start: the API docs server `run_daemon` starts on a thread of its own: one OpenAPI description per running model server built with `utoipa` from a route catalogue, Swagger UI from `utoipa-swagger-ui`, and the forwarder that carries "Try it out" requests to the server. - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, `SandboxToolArg`/`SandboxToolPolicy`, the bridge-snapshot diagnostic printer, and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. - [`common.rs`](../apps/rocmd/src/common.rs) — helpers shared across ≥2 of the other clusters: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, `CommandCapture` and the shared `rocm`-subprocess capture helpers the sandbox and MCP clusters both call (the bounded-execution half of that plumbing lives in [`process.rs`](../crates/rocm-core/src/process.rs) rather than here), and small arg/healthcheck/endpoint-key utilities. A helper earns a place here only once a second still-inline cluster calls it directly; a helper with exactly one caller stays next to that caller until its own cluster's extraction PR, even if it is conceptually similar to something that did move. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the `rocm`-subprocess capture/argv-building helpers behind the MCP tools. @@ -84,6 +86,7 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) — (ROCMAI-85, Phase 7a) the TheRock managed-runtime-environment surface — see the module's own doc comment for what it owns. - [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) — (ROCMAI-152, Phase 7c) the curated model catalog's types, built-in catalog JSON loading, signed external recipe indexes, and artifact cache-marker bookkeeping — see the module's own doc comment for what it owns. - [`model_readiness.rs`](../crates/rocm-core/src/model_readiness.rs) — the `rocm diagnose --model`/`rocm model --verbose` fit assessment: curated-recipe lookup against a host's measured GPU/RAM, shared between the two commands so they cannot disagree about whether a model fits. +- [`model_task.rs`](../crates/rocm-core/src/model_task.rs) — `ModelTask`, what a served model does (chat or speech-to-text): its client route, how readiness probes it, and which commands may use it. - [`net.rs`](../crates/rocm-core/src/net.rs) — (ROCMAI-154, Phase 7b) networking/download/endpoint-readiness primitives. - [`paths.rs`](../crates/rocm-core/src/paths.rs) — (ROCMAI-154, Phase 7b) `AppPaths` and its filesystem-location primitives. - [`process.rs`](../crates/rocm-core/src/process.rs) — (ROCMAI-154, Phase 7b) process-spawn/terminate primitives, including `FileLock`, plus `run_with_timeout`: it drains both pipes on their own threads from the moment the child starts, so output size cannot stall a command into its own timeout, and it bounds how much of a timed-out child's output the error quotes, because that text reaches persisted automation logs. It is not yet the *only* such helper — [`therock.rs`](../apps/rocm/src/therock.rs)'s `wait_with_output_bounded`, [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `run_raw`, and [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs)'s `capture_optional_command_candidate_with_timeout` (plus its Windows-only `capture_diagnostic_command`) still join their reader threads with no time limit, so a grandchild holding the pipe can outlast their timeout; converging them onto `run_with_timeout` is follow-up work to ROCMAI-448 tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623). @@ -156,6 +159,7 @@ These modules are private `mod` declarations — [`lib.rs`](../crates/e2e-report - [`install.rs`](../engines/lemonade/src/install.rs) — embeddable-runtime archive download/verify/extract, cache locking, and manifest persistence. - [`process.rs`](../engines/lemonade/src/process.rs) — `lemond`/llama-server spawn, env/path setup, and readiness/health polling. - [`runtime_dir.rs`](../engines/lemonade/src/runtime_dir.rs) — `XDG_RUNTIME_DIR` tier1/2/3 hardening for the spawned child. +- [`speech.rs`](../engines/lemonade/src/speech.rs) — speech-to-text serving through the `whispercpp` recipe: GPU backend selection, load, and the post-load transcription check. - [`state.rs`](../engines/lemonade/src/state.rs) — service state file I/O, device-policy/GPU-selection parsing, and port/endpoint helpers. [`lib.rs`](../engines/lemonade/src/lib.rs) keeps only CLI/envelope dispatch and the service-lifecycle orchestration that calls across those modules. diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index b7d7e2b406..5310b1ddfb 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -32,6 +32,36 @@ The `lemonade` adapter uses Lemonade embeddable and prefers Lemonade's `llamacpp:rocm` backend, falling back to `llamacpp:vulkan` when ROCm is unsupported. rocm-cli does not use a CPU fallback for this path. +## Model tasks + +An engine advertises which kinds of model it can serve in the `tasks` field of +its capabilities: `chat` (`POST /v1/chat/completions`) and `transcription` +(speech-to-text, `POST /v1/audio/transcriptions`). An engine that does not +send the field — an external plugin written before it existed — is treated as +serving `chat` only. `rocm serve` reads each model's task from its catalog +recipe (a model outside the catalog is a chat model) and refuses, before any +GPU check or engine preparation, to hand a model to an engine that does not +advertise its task. The task is passed to the engine as `task` in the +`ResolveModel` and `Launch` requests, omitted for chat so a request for a chat +model is unchanged. + +`lemonade` advertises `chat` and `transcription`; `vllm` advertises `chat`. + +Lemonade serves a speech-to-text model through its router with the +`whispercpp` recipe. The adapter names the backend explicitly — `rocm`, or +`vulkan` when Lemonade reports ROCm unsupported, never `cpu` — because +Lemonade's `auto` picks whichever whisper.cpp build is installed, which can be +the CPU one. Before reporting ready it checks that Lemonade loaded the model on +that backend and that a test transcription returns HTTP 200. It does not trust +the per-model `device` field of Lemonade's health report: Lemonade 11.5.1 +derives it from the backend name, labelling a ROCm Whisper model `cpu`, and a +Vulkan one `gpu` even where whisper.cpp found no Vulkan device. `rocm serve`'s +usable-AMD-GPU check, which runs before the engine starts, ensures an AMD GPU +is present, but not that a Vulkan whisper.cpp build can use it: where the +Vulkan driver exposes no device, that build may run on the CPU without +reporting it. This is a known gap, shared with the chat path's Vulkan +llama.cpp; the ROCm backend is not affected. + ## Lemonade backend alignment on engine install Lemonade's `resources/backend_versions.json` pins which ROCm SDK version its diff --git a/docs/manual-testing.md b/docs/manual-testing.md index f29df54862..346ae57e64 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -285,6 +285,36 @@ While the log stream is attached, verify detach and stop behave differently: - Re-run the serve command and press `Ctrl+C` instead. The server shuts down and `rocm services` no longer lists it as running. +### Speech-to-text on Lemonade + +Serve the smallest Whisper model and transcribe a recording through the +endpoint the summary prints: + +```powershell +rocm serve whisper-tiny +curl.exe http://127.0.0.1:11435/v1/audio/transcriptions -F model=Whisper-Tiny -F file=@recording.wav +rocm services +``` + +Expected result: + +- The summary's `endpoint` row ends in `/v1/audio/transcriptions` and a `task` + row says `speech-to-text`; there are no time-to-first-token or throughput rows. +- The `curl` call returns JSON whose `text` is the recording's words. +- `rocm services` lists the server with `task: speech-to-text`, its base + `endpoint`, and a `speech-to-text endpoint` line with the same transcription + URL as the summary. +- `rocm chat --provider local --model Whisper-Tiny --prompt hi` is refused and + says the model is a speech-to-text model; no chat request is sent to the server. +- `rocm serve whisper-tiny --engine vllm` is refused before any GPU check and + names `rocm serve whisper-tiny --engine lemonade`. + +Confirm the GPU did the work by watching GPU utilization (`amd-smi metric` on +Linux, Task Manager's GPU graph on Windows) while transcribing a recording of a +minute or more. Do not rely on the `device` field of Lemonade's own +`/api/v1/health`: Lemonade 11.5.1 derives it from the backend name and reports +a ROCm Whisper model as `cpu`. + ## 4. Local Server Records After a managed or foreground serve attempt, inspect local server records: @@ -614,6 +644,30 @@ Expected result: On an 80x24 terminal, repeat the steps up to the dialog: the card shows the outcome (report ready or refusal) without scrolling. +### 4b. API docs + +With a model served by `rocm serve ` (any engine): + +Expected result: + +- The deployment summary has a `docs` row under `endpoint`, and `rocm services + list` shows the same `docs:` link for that server. +- Opening the link in a browser shows the server's routes. Under + `POST /v1/chat/completions`, "Try it out" then "Execute" returns a chat + completion from the served model; the example body already names that model. +- The page loads with the network disconnected (the docs need no CDN). +- For a speech-to-text server (`rocm serve whisper`), the page lists + `POST /v1/audio/transcriptions` and no chat routes; choosing a WAV file under + "Try it out" and pressing "Execute" returns its transcript. +- For a server started with `--require-api-key` (an endpoint key), "Execute" + is refused until the key is entered under "Authorize". +- After `rocm services stop --yes`, reloading the page says the + server is not running, and `rocm services list` prints no docs link. +- With `"api_docs": { "enabled": false }` in `config.json` and the helper + restarted, `rocm serve` prints no docs link. +- With `HTTP_PROXY` set to an address nothing listens on before the helper + starts, "Try it out" still reaches the server. + ## 5. ComfyUI Verification ComfyUI is managed as an app surface. It should start a local web server and diff --git a/docs/testing.md b/docs/testing.md index 30b1486e47..c343d9fc29 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -300,6 +300,45 @@ export ROCM_TEST_APK_REPOS="--repository http://dl-cdn.alpinelinux.org/alpine/v3 --repository http://dl-cdn.alpinelinux.org/alpine/v3.20/community" ``` +## API docs + +The API docs server lives in `rocmd` +([api_docs.rs](../apps/rocmd/src/api_docs.rs)); the link the CLI prints is +built in [api_docs_link.rs](../apps/rocm/src/api_docs_link.rs). Unit tests run a +real docs server against a stand-in model server, covering the route catalogue, +the forwarder (what it passes on and what it drops, and that a configured proxy +does not divert it), stopped and unreachable servers, the loopback `Host` and +`Origin` guard, and a taken or zero docs port. The CLI's tests cover when a +link is and is not printed: + +```bash +cargo test -p rocmd api_docs +cargo test -p rocm --bin rocm api_docs_link +cargo test -p rocm --bin rocm serve_summary +``` + +[service_docs.feature](../tests/e2e-cucumber/features/service_docs.feature) +starts the real `rocm daemon` in the scenario's isolated tree, on a port of its +own set in that tree's `config.json`, and drives the CLI and the docs over HTTP. +`service-docs-03` stops the server with `rocm services stop`, so it is Linux +only (`@requires-os:linux`): + +```bash +cargo xtask e2e -- -n "service-docs-" +``` + +`service-docs-05` covers a speech-to-text server: its docs list the +transcription route with an audio upload and no chat routes, and an upload +through them is transcribed. + +The link `rocm serve` itself prints, and a try-it request through it to a real +engine, are checked by the GPU serve canaries `serve-05` (vLLM) and `serve-07` +(Lemonade), and for a speech-to-text server by `serve-27`, which only run on +GPU lanes. On Linux Lemonade lanes `serve-07` is +an expected failure for an unrelated known issue, so there a broken docs link +would not turn the lane red: the Lemonade check is effective on the Windows +Lemonade lanes, and `serve-05` on the vLLM lanes. + ## CI test selection On pull requests, CI does not test the whole workspace. The `test` job runs @@ -1906,6 +1945,30 @@ The gate itself is held by unit tests in neighbours) and by the Windows manual check in [manual-testing.md](manual-testing.md), in the Ryzen 4000/5000 section. +## Speech-to-text Serving + +Focused coverage for model tasks and speech-to-text serving: + +```bash +cargo test -p rocm-core model_task +cargo test -p rocm-core speech_to_text +cargo test -p rocm-engine-protocol capabilities_from_an_engine_without_tasks +cargo test -p rocm-engine-lemonade speech:: +cargo test -p rocm-engine-vllm refuse_a_speech_to_text_model +cargo test -p rocm --bin rocm speech_to_text +cargo test -p rocm-dash-tui speech_to_text +``` + +The user-facing behaviour is in `serve-26` to `serve-28`, `chat-10`, and +`bench-07`. All but `serve-27` run on every lane: `serve-26` needs no server at +all, and the others run against a mock speech-to-text server. `serve-27` serves +Whisper-Tiny on a real GPU and needs the Lemonade GPU lanes +(`@requires-gpu @requires-engine:lemonade`). + +Running a subset with `cargo xtask e2e -- -n ` bypasses the +capability filter, so a `@requires-gpu` scenario the pattern matches runs, and +fails, on a host without a GPU. Read its failure as "not applicable here". + ## Model Fit Preflight `rocm diagnose --model ` answers whether a curated model will run on this diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index ac2da648a6..2266c2da21 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -813,16 +813,28 @@ fn best_llamacpp_backend_for_host(manifest: &LemonadeInstallManifest) -> Result< /// completed download when its connection to lemond is interrupted, and a second /// call can reuse the backend cache immediately without a delay. pub(crate) fn install_llamacpp_backend_with_retry( + install: impl FnMut() -> Result<()>, +) -> Result<()> { + install_backend_with_retry( + install, + "Lemonade backend installation failed again; run `rocm engines install \ + lemonade --reinstall` and then retry `rocm serve`", + ) +} + +/// Install a backend, retrying once; see [`install_llamacpp_backend_with_retry`]. +/// `advice` is the remedy shown when the retry fails too. It has to name a step +/// that actually installs *this* backend: `rocm engines install lemonade +/// --reinstall` only ever reinstalls the llama.cpp one. +pub(crate) fn install_backend_with_retry( mut install: impl FnMut() -> Result<()>, + advice: &'static str, ) -> Result<()> { match install() { Ok(()) => Ok(()), Err(first_error) => { eprintln!("Lemonade backend installation failed; retrying once: {first_error:#}"); - install().with_context(|| { - "Lemonade backend installation failed again; run `rocm engines install \ - lemonade --reinstall` and then retry `rocm serve`" - }) + install().context(advice) } } } @@ -842,7 +854,7 @@ pub(crate) fn ensure_best_llamacpp_backend( The GPU-required policy does not fall back to CPU; install a ROCm or Vulkan \ backend (e.g. `rocm engines install lemonade`) or verify the GPU driver with \ `rocm examine`.", - describe_llamacpp_backends(&backends) + describe_backend_statuses(&backends) ); }; // Record the attempted backend before the fallible install step below, so a caller @@ -863,15 +875,31 @@ pub(crate) fn ensure_best_llamacpp_backend( /// Parse `lemonade backends` table output into `(backend, status)` pairs for the /// `llamacpp` recipe only. Status is one of `installed`/`installable`/`unsupported`. fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { - const RECIPES: [&str; 8] = [ + parse_backend_statuses(output, LLAMACPP_RECIPE) +} + +/// Parse `lemonade backends` table output into `(backend, status)` pairs for one +/// recipe. Status is one of `installed`/`installable`/`unsupported`. +/// +/// Every recipe the table can list must be named here: a row whose first token is +/// not a known recipe is read as another backend of the recipe above it, so an +/// unknown recipe's rows would be attributed to its neighbour. +pub(crate) fn parse_backend_statuses(output: &str, recipe: &str) -> Vec<(String, String)> { + const RECIPES: [&str; 14] = [ + "acestep", + "embeddings", "flm", "kokoro", "llamacpp", + "moonshine", + "onnxruntime", + "openmoss", "ryzenai-llm", "sd-cpp", + "thinksound", + "trellis", "vllm", "whispercpp", - "embeddings", ]; const STATUSES: [&str; 3] = ["installed", "installable", "unsupported"]; let mut current_recipe = ""; @@ -898,7 +926,7 @@ fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { } else { (first, &tokens[1..]) }; - if current_recipe != LLAMACPP_RECIPE { + if current_recipe != recipe { continue; } if let Some(status) = rest.iter().find(|t| STATUSES.contains(t)) { @@ -911,10 +939,19 @@ fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { /// Choose the highest-priority backend Lemonade considers usable (installed or /// installable). Returns `(backend, already_installed)`. fn select_best_llamacpp_backend(backends: &[(String, String)]) -> Option<(String, bool)> { - for candidate in LLAMACPP_BACKEND_PRIORITY { + select_best_backend(backends, &LLAMACPP_BACKEND_PRIORITY) +} + +/// The first backend in `priority` that Lemonade reports as installed or +/// installable. Returns `(backend, already_installed)`. +pub(crate) fn select_best_backend( + backends: &[(String, String)], + priority: &[&str], +) -> Option<(String, bool)> { + for candidate in priority { if let Some((name, status)) = backends .iter() - .find(|(b, s)| b == candidate && (s == "installed" || s == "installable")) + .find(|(b, s)| b == *candidate && (s == "installed" || s == "installable")) { return Some((name.clone(), status == "installed")); } @@ -922,7 +959,7 @@ fn select_best_llamacpp_backend(backends: &[(String, String)]) -> Option<(String None } -fn describe_llamacpp_backends(backends: &[(String, String)]) -> String { +pub(crate) fn describe_backend_statuses(backends: &[(String, String)]) -> String { if backends.is_empty() { return "none reported".to_owned(); } @@ -933,7 +970,7 @@ fn describe_llamacpp_backends(backends: &[(String, String)]) -> String { .join(", ") } -fn run_lemonade_backends_list( +pub(crate) fn run_lemonade_backends_list( manifest: &LemonadeInstallManifest, process_env: &LemonadeProcessEnvironment, ) -> Result { diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 7f8fac2934..eb6516eb79 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -9,8 +9,8 @@ use rocm_engine_protocol::{ DetectRequest, DetectResponse, DevicePolicy, EndpointRequest, EndpointResponse, EngineCapabilities, EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, HealthcheckRequest, HealthcheckResponse, InstallRequest, InstallResponse, LaunchRequest, - LaunchResponse, LogsRequest, LogsResponse, ResolveModelRequest, ResolveModelResponse, - StopRequest, StopResponse, + LaunchResponse, LogsRequest, LogsResponse, ModelTask, ResolveModelRequest, + ResolveModelResponse, StopRequest, StopResponse, }; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -23,6 +23,7 @@ mod direct_llama; mod install; mod process; mod runtime_dir; +mod speech; mod state; #[cfg(test)] mod test_support; @@ -107,6 +108,9 @@ enum CommandKind { engine_recipe_json: Option, #[arg(long)] gpu: Option, + /// What the model does: `chat` (default) or `transcription`. + #[arg(long, default_value = "chat")] + task: String, }, } @@ -123,6 +127,7 @@ pub(crate) struct ServeHttpRequest { pub(crate) state_path: PathBuf, pub(crate) log_path: Option, pub(crate) engine_recipe: Option, + pub(crate) task: ModelTask, } pub fn run_cli() -> Result<()> { @@ -146,6 +151,7 @@ pub fn run_cli() -> Result<()> { device_policy: None, recipe_override: None, engine_recipe: None, + task: ModelTask::Chat, })?)?; } CommandKind::Launch { @@ -170,6 +176,7 @@ pub fn run_cli() -> Result<()> { endpoint_mode: Some("openai".to_owned()), engine_recipe: None, gpu_selection: crate::state::parse_gpu_selection_arg(gpu.as_deref())?, + task: ModelTask::Chat, })?)?, CommandKind::Stdio => { let envelope = read_request()?; @@ -187,6 +194,7 @@ pub fn run_cli() -> Result<()> { log_path, engine_recipe_json, gpu, + task, } => serve_http(ServeHttpRequest { service_id, model_ref, @@ -199,6 +207,7 @@ pub fn run_cli() -> Result<()> { state_path, log_path, engine_recipe: crate::state::parse_engine_recipe_json(engine_recipe_json)?, + task: ModelTask::parse(&task)?, })?, } Ok(()) @@ -221,6 +230,7 @@ pub fn builtin_serve_http( state_path: PathBuf, log_path: Option, engine_recipe: Option, + task: ModelTask, ) -> Result<()> { serve_http(ServeHttpRequest { service_id, @@ -234,6 +244,7 @@ pub fn builtin_serve_http( state_path, log_path, engine_recipe, + task, }) } @@ -306,6 +317,7 @@ fn capabilities() -> EngineCapabilities { quantized_models: "GGUF through Lemonade llama.cpp (ROCm or Vulkan GPU backend auto-selected)".to_owned(), reasoning_parser: false, + tasks: vec![ModelTask::Chat, ModelTask::Transcription], } } @@ -407,9 +419,17 @@ fn install_response(request: InstallRequest) -> Result { } fn resolve_model_response(request: ResolveModelRequest) -> Result { + request.task.ensure_servable()?; let device_policy = crate::state::normalize_device_policy(request.device_policy)?; let engine_recipe = crate::state::accepted_engine_recipe(request.engine_recipe)?; let canonical_model_id = crate::state::resolve_lemonade_model_ref(&request.model_ref); + if request.task == ModelTask::Transcription { + return Ok(crate::speech::resolve_transcription_model( + canonical_model_id, + device_policy, + engine_recipe, + )); + } Ok(ResolveModelResponse { canonical_model_id, task: "chat-completions".to_owned(), @@ -434,6 +454,7 @@ fn resolve_model_response(request: ResolveModelRequest) -> Result Result { + request.task.ensure_servable()?; rocm_core::require_nonempty(&request.service_id, "service_id")?; rocm_core::require_nonempty(&request.model_ref, "model_ref")?; request.device_policy = Some(crate::state::normalize_device_policy( @@ -467,6 +488,7 @@ fn launch_service(mut request: LaunchRequest) -> Result { state_path: state_path.clone(), log_path: Some(log_path.clone()), engine_recipe: request.engine_recipe.clone(), + task: request.task, }; let current_exe = std::env::current_exe().context("failed to discover current Lemonade engine binary")?; @@ -510,6 +532,9 @@ fn serve_http(mut request: ServeHttpRequest) -> Result<()> { process_env.gpu_indices = request.gpu_indices.clone(); let log_path = request.log_path.as_deref(); crate::state::write_running_state(&request, &runtime, std::process::id(), None, "starting")?; + if request.task == ModelTask::Transcription { + return crate::speech::serve_transcription(&request, &runtime, &process_env, log_path); + } // A canonical Hugging Face checkpoint (owner/repo:variant) cannot be served under its // exact name through Lemonade's model router (Lemonade renames registered models and // its registry has several naming quirks). Download the GGUF and run a packaged @@ -700,12 +725,22 @@ fn healthcheck_service(request: HealthcheckRequest) -> Result { + crate::process::query_loaded_model_endpoint(endpoint, &model_ref, &backend) + } + ModelTask::Transcription => { + crate::speech::query_loaded_transcription_model(endpoint, &model_ref, &backend) + } + // A task this build cannot name was written by a newer one + // (`task_from_state`); it has no probe here, so it is never + // reported ready. + ModelTask::Unknown => Ok(false), }) .transpose() .unwrap_or(None) @@ -745,12 +780,10 @@ fn endpoint_response(request: EndpointRequest) -> Result { Ok(EndpointResponse { endpoint_url, api_style: "openai".to_owned(), - supported_routes: vec![ - "/v1/health".to_owned(), - "/v1/models".to_owned(), - "/v1/chat/completions".to_owned(), - "/v1/completions".to_owned(), - ], + supported_routes: rocm_engine_protocol::supported_routes( + "/v1/health", + crate::state::task_from_state(Some(&state)), + ), }) } @@ -819,6 +852,20 @@ fn print_json(value: &T) -> Result<()> { mod tests { use super::*; + #[test] + fn a_task_from_a_newer_rocm_cli_is_not_resolved_as_any_known_task() { + let error = resolve_model_response(ResolveModelRequest { + model_ref: "future-model".to_owned(), + runtime_id: None, + device_policy: None, + recipe_override: None, + engine_recipe: None, + task: ModelTask::Unknown, + }) + .expect_err("an unknown task is refused, not served as chat"); + assert!(error.to_string().contains("a newer rocm-cli"), "{error}"); + } + #[test] fn stop_scope_targets_only_the_recorded_lemonade_server() { assert_eq!(STOP_SCOPE, rocm_core::KillScope::Single); diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs index 84f6e656aa..d67d3c334b 100644 --- a/engines/lemonade/src/process.rs +++ b/engines/lemonade/src/process.rs @@ -320,6 +320,18 @@ pub(crate) fn run_lemonade_backend_install( port: u16, backend: &str, process_env: &LemonadeProcessEnvironment, +) -> Result<()> { + run_lemonade_recipe_backend_install(manifest, host, port, LLAMACPP_RECIPE, backend, process_env) +} + +/// `lemonade backends install :` against a running server. +pub(crate) fn run_lemonade_recipe_backend_install( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + recipe: &str, + backend: &str, + process_env: &LemonadeProcessEnvironment, ) -> Result<()> { let mut command = ProcessCommand::new(&manifest.lemonade); command @@ -329,7 +341,7 @@ pub(crate) fn run_lemonade_backend_install( .arg(port.to_string()) .arg("backends") .arg("install") - .arg(format!("{LLAMACPP_RECIPE}:{backend}")) + .arg(format!("{recipe}:{backend}")) .stdin(Stdio::null()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()); @@ -403,17 +415,25 @@ pub(crate) fn run_lemonade_model_load( engine_recipe: Option<&EngineRecipeHint>, log_path: Option<&Path>, process_env: &LemonadeProcessEnvironment, +) -> Result<()> { + run_lemonade_load_command( + manifest, + &lemonade_model_load_args(host, port, model_ref, backend, engine_recipe), + log_path, + process_env, + ) +} + +/// Run a `lemonade ... load ...` invocation, sending its output to the service +/// log when there is one. +pub(crate) fn run_lemonade_load_command( + manifest: &LemonadeInstallManifest, + args: &[String], + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, ) -> Result<()> { let mut command = ProcessCommand::new(&manifest.lemonade); - command - .args(lemonade_model_load_args( - host, - port, - model_ref, - backend, - engine_recipe, - )) - .stdin(Stdio::null()); + command.args(args).stdin(Stdio::null()); if let Some(log_path) = log_path { let log = fs::OpenOptions::new() .create(true) @@ -702,7 +722,7 @@ fn lemonade_model_load_args( args } -fn query_health_json(host: &str, port: u16) -> Result { +pub(crate) fn query_health_json(host: &str, port: u16) -> Result { let endpoint = format_http_base_url(host, port); let endpoint_api_key = rocm_engine_protocol::resolve_endpoint_api_key(); let body = http_get_text_with_auth( diff --git a/engines/lemonade/src/speech.rs b/engines/lemonade/src/speech.rs new file mode 100644 index 0000000000..44ac54c6ab --- /dev/null +++ b/engines/lemonade/src/speech.rs @@ -0,0 +1,503 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Speech-to-text serving through Lemonade's `whispercpp` recipe. +//! +//! A transcription model always runs behind Lemonade's router (`lemond`): the +//! direct `llama-server` path the chat flow can take on Linux has no audio +//! route. The backend is chosen the same way as for chat — ROCm first, Vulkan +//! when Lemonade reports ROCm unsupported, never CPU — and is named explicitly +//! on load, because Lemonade's `auto` resolves to whatever whisper.cpp build is +//! installed, which can be the CPU one. +//! +//! What this can and cannot verify: Lemonade 11.5.1 reports a loaded model's +//! `device`, but derives it from the backend *name* — a `rocm` Whisper is +//! reported as `cpu`, and a `vulkan` one as `gpu` even where whisper.cpp found +//! no Vulkan device and ran on the CPU. So the field proves nothing either way +//! and is not consulted. What is checked is the backend Lemonade actually loaded +//! (`recipe_options.whispercpp_backend`), exactly as the chat path checks +//! `llamacpp_backend`. `rocm serve`'s usable-AMD-GPU gate, which runs before +//! any engine is started, ensures an AMD GPU is present, but not that a Vulkan +//! whisper.cpp build can use it: with no Vulkan device it may run on the CPU +//! unreported. That gap is shared with the chat path's Vulkan llama.cpp and is +//! documented in docs/engine-plugins.md. + +use anyhow::{Context, Result, anyhow, bail}; +use rocm_core::format_http_base_url; +use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; +use serde_json::{Value, json}; +use std::path::Path; +use std::time::Duration; + +use crate::backend_alignment::LemonadeRuntime; +use crate::process::LemonadeProcessEnvironment; +use crate::{DEFAULT_HOST, ServeHttpRequest}; + +/// Lemonade's speech-to-text recipe. +pub(crate) const WHISPERCPP_RECIPE: &str = "whispercpp"; + +/// GPU backends only, best first. `cpu` is deliberately absent: under the +/// GPU-required policy a host without a usable GPU backend fails instead of +/// transcribing on the CPU. +const WHISPERCPP_BACKEND_PRIORITY: [&str; 2] = ["rocm", "vulkan"]; + +/// Shown when the whisper.cpp backend fails to install twice in a row. The +/// download is retried by every serve that finds the backend missing, so +/// running the same command again is the step that installs it. +const WHISPERCPP_INSTALL_ADVICE: &str = "Lemonade could not install its speech-to-text \ + (whisper.cpp) backend; check the network connection and run the same `rocm serve` \ + command again, which retries the download"; + +/// How long `lemond` gets to answer its status query after it is spawned. +const LEMOND_STARTUP_TIMEOUT: Duration = Duration::from_secs(30); + +pub(crate) fn resolve_transcription_model( + canonical_model_id: String, + device_policy: DevicePolicy, + engine_recipe: Option, +) -> ResolveModelResponse { + ResolveModelResponse { + canonical_model_id, + task: rocm_engine_protocol::ModelTask::Transcription + .as_str() + .to_owned(), + source: "lemonade".to_owned(), + revision: "main".to_owned(), + loader: WHISPERCPP_RECIPE.to_owned(), + trust_remote_code: false, + chat_template_mode: "none".to_owned(), + dtype: "ggml".to_owned(), + device_policy, + estimated_memory: "under 2 GiB for Whisper Large-v3-Turbo; well under 1 GiB for the \ + smaller Whisper models" + .to_owned(), + launch_defaults: json!({ + "host": DEFAULT_HOST, + "port": rocm_core::DEFAULT_LOCAL_PORT, + "endpoint_mode": "openai" + }), + engine_recipe, + warnings: vec![ + "Lemonade runs Whisper on its ROCm whisper.cpp backend, or Vulkan when ROCm is \ + unsupported on this GPU; its CPU build is never selected" + .to_owned(), + ], + } +} + +/// Serve a speech-to-text model until the router exits. +pub(crate) fn serve_transcription( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + process_env: &LemonadeProcessEnvironment, + log_path: Option<&Path>, +) -> Result<()> { + let manifest = &runtime.manifest; + let mut child = + crate::process::spawn_lemond(manifest, &request.host, request.port, log_path, process_env)?; + crate::state::write_running_state( + request, + runtime, + std::process::id(), + Some(child.id()), + "running", + )?; + let prepared = prepare_transcription_model(request, runtime, process_env, log_path); + let backend = match prepared { + Ok(backend) => backend, + Err(error) => { + let _ = crate::state::terminate_pid(child.id(), true); + let _ = child.wait(); + return Err(error); + } + }; + crate::state::merge_json_state( + &request.state_path, + &json!({ + "status": "ready", + "server_pid": child.id(), + "server_start_ticks": rocm_core::process_start_ticks(child.id()), + "backend_state": "ready", + "backend_requested": backend, + "load_response": { + "status": "loaded", + "method": "lemonade-cli", + "model_name": request.model_ref, + "whispercpp_backend": backend + }, + }), + )?; + let status = child.wait().context("failed waiting for Lemonade server")?; + crate::state::mark_json_status( + &request.state_path, + if status.success() { + "stopped" + } else { + "failed" + }, + )?; + if status.success() { + Ok(()) + } else { + bail!("Lemonade server exited with status {status}") + } +} + +/// Bring the router to the point of serving: pick and install a GPU backend, +/// load the model on it, and prove one transcription succeeds. Returns the +/// backend in use. +fn prepare_transcription_model( + request: &ServeHttpRequest, + runtime: &LemonadeRuntime, + process_env: &LemonadeProcessEnvironment, + log_path: Option<&Path>, +) -> Result { + let manifest = &runtime.manifest; + crate::process::wait_for_lemonade_cli_status( + manifest, + &request.host, + request.port, + LEMOND_STARTUP_TIMEOUT, + log_path, + process_env, + ) + .context("Lemonade server did not become ready")?; + let listing = crate::backend_alignment::run_lemonade_backends_list(manifest, process_env)?; + let backend = select_whispercpp_backend(&listing)?; + if backend.already_installed { + eprintln!( + "Using installed Lemonade {WHISPERCPP_RECIPE}:{} backend.", + backend.name + ); + } else { + eprintln!( + "Installing Lemonade {WHISPERCPP_RECIPE}:{} backend...", + backend.name + ); + crate::backend_alignment::install_backend_with_retry( + || { + crate::process::run_lemonade_recipe_backend_install( + manifest, + &request.host, + request.port, + WHISPERCPP_RECIPE, + &backend.name, + process_env, + ) + }, + WHISPERCPP_INSTALL_ADVICE, + )?; + } + crate::process::run_lemonade_load_command( + manifest, + &transcription_load_args( + &request.host, + request.port, + &request.model_ref, + &backend.name, + ), + log_path, + process_env, + ) + .with_context(|| { + format!( + "failed to load {} with Lemonade {WHISPERCPP_RECIPE}:{}", + request.model_ref, backend.name + ) + })?; + let health = crate::process::query_health_json(&request.host, request.port)?; + if !health_has_transcription_model(&health, &request.model_ref, &backend.name) { + bail!( + "Lemonade load completed but the server did not report {} loaded on its \ + {WHISPERCPP_RECIPE}:{} backend", + request.model_ref, + backend.name + ); + } + let status = rocm_core::openai_transcription_status( + &format_http_base_url(&request.host, request.port), + &request.model_ref, + rocm_engine_protocol::resolve_endpoint_api_key().as_deref(), + rocm_core::INFERENCE_PROBE_TIMEOUT, + ) + .context("the post-load transcription smoke test did not complete")?; + if status != 200 { + bail!( + "{} loaded, but a test transcription returned HTTP {status}", + request.model_ref + ); + } + Ok(backend.name) +} + +#[derive(Debug, PartialEq, Eq)] +struct SelectedBackend { + name: String, + already_installed: bool, +} + +/// Pick the whisper.cpp backend from `lemonade backends` output. +fn select_whispercpp_backend(listing: &str) -> Result { + let statuses = crate::backend_alignment::parse_backend_statuses(listing, WHISPERCPP_RECIPE); + let (name, already_installed) = + crate::backend_alignment::select_best_backend(&statuses, &WHISPERCPP_BACKEND_PRIORITY) + .ok_or_else(|| { + anyhow!( + "Lemonade reports no supported GPU speech-to-text backend for this host \ + (status: {}). The GPU-required policy does not fall back to CPU; check \ + the GPU driver with `rocm examine`.", + crate::backend_alignment::describe_backend_statuses(&statuses) + ) + })?; + Ok(SelectedBackend { + name, + already_installed, + }) +} + +fn transcription_load_args(host: &str, port: u16, model_ref: &str, backend: &str) -> Vec { + vec![ + "--host".to_owned(), + host.to_owned(), + "--port".to_owned(), + port.to_string(), + "load".to_owned(), + model_ref.to_owned(), + "--whispercpp".to_owned(), + backend.to_owned(), + "--save-options".to_owned(), + ] +} + +/// Whether Lemonade reports `model_ref` loaded through `whispercpp` on exactly +/// `backend`. +fn health_has_transcription_model(health: &Value, model_ref: &str, backend: &str) -> bool { + let model_ref = model_ref.trim(); + if model_ref.is_empty() { + return false; + } + health + .get("all_models_loaded") + .and_then(Value::as_array) + .into_iter() + .flatten() + .any(|model| { + let str_field = |key: &str| model.get(key).and_then(Value::as_str); + str_field("model_name").is_some_and(|name| name.eq_ignore_ascii_case(model_ref)) + && str_field("recipe") == Some(WHISPERCPP_RECIPE) + && model + .get("recipe_options") + .and_then(|options| options.get("whispercpp_backend")) + .and_then(Value::as_str) + == Some(backend) + }) +} + +/// The healthcheck's "listed" signal for a speech-to-text service. +pub(crate) fn query_loaded_transcription_model( + endpoint_url: &str, + model_ref: &str, + backend: &str, +) -> Result { + let (host, port) = crate::state::parse_http_endpoint(endpoint_url) + .with_context(|| format!("unsupported endpoint URL `{endpoint_url}`"))?; + let health = crate::process::query_health_json(&host, port)?; + Ok(health_has_transcription_model(&health, model_ref, backend)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `lemonade backends --all` output captured from Lemonade 11.5.1 on a + /// Linux host where ROCm is unsupported, trimmed to the rows around + /// `whispercpp` plus the recipes 11.5.1 added before it. + const BACKENDS_NO_ROCM: &str = "\ +Recipe Backend Status Message/Version Action +---------------------------------------------------------------------------------------------------- +acestep cuda unsupported Unsupported GPU - + rocm unsupported Unsupported GPU - + vulkan installable Backend is supported but not installed. lemonade backends install acestep:vulkan +kokoro cpu installable Backend is supported but not installed. lemonade backends install kokoro:cpu +llamacpp cpu installable Backend is supported but not installed. lemonade backends install llamacpp:cpu + rocm unsupported Unsupported GPU - + vulkan installed b9747 - +openmoss rocm unsupported Unsupported GPU - + vulkan installable Backend is supported but not installed. lemonade backends install openmoss:vulkan +vllm rocm unsupported Unsupported GPU - +whispercpp cpu installed v1.8.4 - + metal unsupported Requires macOS - + npu unsupported Requires Windows - + rocm unsupported Unsupported GPU - + vulkan installable Backend is supported but not installed. lemonade backends install whispercpp:vulkan +"; + + #[test] + fn rocm_unsupported_selects_vulkan_even_when_the_cpu_build_is_installed() { + assert_eq!( + select_whispercpp_backend(BACKENDS_NO_ROCM).expect("vulkan is usable"), + SelectedBackend { + name: "vulkan".to_owned(), + already_installed: false, + } + ); + } + + #[test] + fn rocm_is_preferred_when_lemonade_supports_it() { + let listing = BACKENDS_NO_ROCM.replace( + " rocm unsupported Unsupported GPU -\n vulkan installable Backend is supported but not installed. lemonade backends install whispercpp:vulkan", + " rocm installed v1.8.4 -\n vulkan installable Backend is supported but not installed. lemonade backends install whispercpp:vulkan", + ); + assert_eq!( + select_whispercpp_backend(&listing).expect("rocm is usable"), + SelectedBackend { + name: "rocm".to_owned(), + already_installed: true, + } + ); + } + + #[test] + fn only_a_cpu_backend_is_refused_rather_than_used() { + let listing = "\ +Recipe Backend Status Message/Version Action +whispercpp cpu installed v1.8.4 - + rocm unsupported Unsupported GPU - + vulkan unsupported Unsupported GPU - +"; + let error = select_whispercpp_backend(listing) + .expect_err("CPU must never be selected") + .to_string(); + assert!(error.contains("does not fall back to CPU"), "{error}"); + assert!(error.contains("cpu=installed"), "{error}"); + } + + #[test] + fn rows_of_recipes_listed_before_whispercpp_are_not_mistaken_for_its_backends() { + // `acestep` and `openmoss` are 11.5.1 recipes with a usable `vulkan` + // row; were they unknown to the parser, those rows would be read as + // continuation rows of the recipe above them. + let statuses = + crate::backend_alignment::parse_backend_statuses(BACKENDS_NO_ROCM, WHISPERCPP_RECIPE); + assert_eq!( + statuses, + vec![ + ("cpu".to_owned(), "installed".to_owned()), + ("metal".to_owned(), "unsupported".to_owned()), + ("npu".to_owned(), "unsupported".to_owned()), + ("rocm".to_owned(), "unsupported".to_owned()), + ("vulkan".to_owned(), "installable".to_owned()), + ] + ); + } + + #[test] + fn rows_of_recipes_listed_after_llamacpp_are_not_attached_to_it() { + // 11.5.1 lists `openmoss` and `vllm` after `llamacpp`. Before those + // recipes were known, their rows read as more llama.cpp backends. + let statuses = crate::backend_alignment::parse_backend_statuses( + BACKENDS_NO_ROCM, + crate::LLAMACPP_RECIPE, + ); + assert_eq!( + statuses, + vec![ + ("cpu".to_owned(), "installable".to_owned()), + ("rocm".to_owned(), "unsupported".to_owned()), + ("vulkan".to_owned(), "installed".to_owned()), + ] + ); + } + + #[test] + fn a_failed_whisper_backend_install_is_retried_once_then_names_a_working_remedy() { + let mut attempts = 0; + let error = crate::backend_alignment::install_backend_with_retry( + || { + attempts += 1; + anyhow::bail!("download interrupted") + }, + WHISPERCPP_INSTALL_ADVICE, + ) + .expect_err("both attempts fail"); + assert_eq!(attempts, 2); + assert!( + error + .to_string() + .contains("run the same `rocm serve` command again"), + "{error}" + ); + // The remedy holds because a serve that finds the backend missing + // installs it: an `installable` backend is selected as not yet + // installed, which is what sends `prepare_transcription_model` down the + // install path again. + assert!( + !select_whispercpp_backend(BACKENDS_NO_ROCM) + .expect("vulkan is usable") + .already_installed + ); + } + + #[test] + fn load_names_the_backend_explicitly_instead_of_auto() { + assert_eq!( + transcription_load_args("127.0.0.1", 13305, "Whisper-Tiny", "rocm"), + vec![ + "--host", + "127.0.0.1", + "--port", + "13305", + "load", + "Whisper-Tiny", + "--whispercpp", + "rocm", + "--save-options", + ] + ); + } + + /// `all_models_loaded` as Lemonade 11.5.1 reported it for Whisper-Tiny + /// loaded with `--whispercpp vulkan` (captured, then trimmed). + fn loaded(backend: &str, device: &str) -> Value { + json!({ + "all_models_loaded": [{ + "model_name": "Whisper-Tiny", + "recipe": "whispercpp", + "device": device, + "type": "transcription", + "recipe_options": {"ctx_size": 4096, "whispercpp_backend": backend} + }] + }) + } + + #[test] + fn loaded_model_must_be_on_the_requested_backend() { + assert!(health_has_transcription_model( + &loaded("vulkan", "gpu"), + "Whisper-Tiny", + "vulkan" + )); + assert!( + !health_has_transcription_model(&loaded("cpu", "cpu"), "Whisper-Tiny", "vulkan"), + "a model Lemonade loaded on its CPU build is not the one we asked for" + ); + assert!(!health_has_transcription_model( + &loaded("vulkan", "gpu"), + "Whisper-Base", + "vulkan" + )); + } + + #[test] + fn the_device_label_is_not_trusted_either_way() { + // Lemonade 11.5.1 labels a ROCm whisper.cpp model `cpu`; the backend + // decides, not the label. + assert!(health_has_transcription_model( + &loaded("rocm", "cpu"), + "Whisper-Tiny", + "rocm" + )); + } +} diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index 6df1ded86d..87a0a9246e 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, format_http_base_url}; +use rocm_core::{AppPaths, ModelTask, format_http_base_url}; use rocm_engine_protocol::{ DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EngineDeviceAvailability, EngineRecipeHint, GpuSelection, @@ -46,6 +46,7 @@ pub(crate) fn inference_verified( return false; }; rocm_core::engine_state_inference_verified( + task_from_state(state), state_path, state, &format_http_base_url(&host, port), @@ -54,6 +55,18 @@ pub(crate) fn inference_verified( ) } +/// The task recorded in service state. State written before the task existed +/// belongs to a chat service; a value this build cannot parse was written by a +/// newer one and reads as [`ModelTask::Unknown`], never as chat, so the +/// service is not probed as a chat model. +pub(crate) fn task_from_state(state: Option<&Value>) -> ModelTask { + state + .and_then(|value| value_string(value, "task")) + .map_or(ModelTask::Chat, |task| { + ModelTask::parse(&task).unwrap_or(ModelTask::Unknown) + }) +} + /// The device string reported once a model is loaded. Reflects the backend that /// actually ran (`rocm`, `vulkan`, …) and the pinned GPU ordinal when known, /// rather than a hardcoded value, so the report matches observed execution. @@ -106,6 +119,9 @@ pub(crate) fn serve_http_command_args(request: &ServeHttpRequest) -> Vec serde_json::to_string(engine_recipe).expect("engine recipe serializes"), ]); } + if !request.task.is_chat() { + args.extend(["--task".to_owned(), request.task.as_str().to_owned()]); + } args } @@ -161,6 +177,7 @@ pub(crate) fn write_running_state( "runtime_executable": runtime.manifest.lemond, "log_path": request.log_path.as_ref().map(|path| path.display().to_string()), "engine_recipe": request.engine_recipe, + "task": request.task, "started_at_unix_ms": current_unix_millis(), // Kernel start-times captured while each PID is alive. Paired with the // PIDs, they identify these exact processes across PID recycling so a @@ -737,6 +754,7 @@ mod tests { state_path: PathBuf::from("state.json"), log_path: Some(PathBuf::from("service.log")), engine_recipe: None, + task: ModelTask::Chat, }; let args = serve_http_command_args(&request); assert!(args.contains(&"--runtime-id".to_owned())); @@ -744,5 +762,38 @@ mod tests { assert!(args.contains(&"--env-id".to_owned())); assert!(args.contains(&"env".to_owned())); assert!(!args.iter().any(|arg| arg == "cpu")); + assert!( + !args.contains(&"--task".to_owned()), + "a chat launch keeps the argument list it always had" + ); + + let transcription = ServeHttpRequest { + task: ModelTask::Transcription, + ..request + }; + let args = serve_http_command_args(&transcription); + let task_at = args + .iter() + .position(|arg| arg == "--task") + .expect("a speech-to-text launch must carry its task to the child"); + assert_eq!(args[task_at + 1], "transcription"); + } + + #[test] + fn state_without_a_task_belongs_to_a_chat_service() { + assert_eq!(task_from_state(None), ModelTask::Chat); + assert_eq!(task_from_state(Some(&json!({}))), ModelTask::Chat); + assert_eq!( + task_from_state(Some(&json!({"task": "transcription"}))), + ModelTask::Transcription + ); + } + + #[test] + fn a_task_this_build_cannot_name_is_not_read_as_chat() { + assert_eq!( + task_from_state(Some(&json!({"task": "text-to-speech"}))), + ModelTask::Unknown + ); } } diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index 6344d6317c..468c1b2b11 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -151,6 +151,7 @@ pub fn run_cli() -> Result<()> { .transpose()?, recipe_override: None, engine_recipe: None, + task: rocm_engine_protocol::ModelTask::Chat, }, )?)?, CommandKind::Launch { @@ -175,6 +176,7 @@ pub fn run_cli() -> Result<()> { endpoint_mode: Some("openai".to_owned()), engine_recipe: None, gpu_selection: crate::process::parse_gpu_selection_arg(gpu.as_deref())?, + task: rocm_engine_protocol::ModelTask::Chat, })?)?, CommandKind::Stdio => { let envelope = read_request()?; @@ -359,6 +361,7 @@ pub(crate) fn capabilities() -> EngineCapabilities { tool_calling: false, quantized_models: "vllm-supported".to_owned(), reasoning_parser: false, + tasks: vec![rocm_engine_protocol::ModelTask::Chat], } } diff --git a/engines/vllm/src/process.rs b/engines/vllm/src/process.rs index 2b86bac4e7..ddbea0b4bc 100644 --- a/engines/vllm/src/process.rs +++ b/engines/vllm/src/process.rs @@ -34,7 +34,21 @@ const STOP_SCOPE: rocm_core::KillScope = rocm_core::KillScope::Tree; /// Default time to wait for vLLM to report readiness before giving up. const DEFAULT_VLLM_READY_TIMEOUT: Duration = Duration::from_mins(5); +/// rocm-cli serves only chat models through vLLM. `rocm serve` checks the +/// engine's advertised tasks before it gets here; this is the engine's own +/// backstop for a caller that skips that check. +fn require_chat_task(task: rocm_engine_protocol::ModelTask) -> Result<()> { + if task.is_chat() { + return Ok(()); + } + bail!( + "the vLLM engine in rocm-cli serves chat models only; it cannot serve a {} model", + task.display_name() + ) +} + pub(crate) fn resolve_model_response(request: ResolveModelRequest) -> Result { + require_chat_task(request.task)?; let device_policy = normalize_vllm_device_policy(request.device_policy)?; let engine_recipe = accepted_engine_recipe(request.engine_recipe)?; Ok(ResolveModelResponse { @@ -93,6 +107,7 @@ pub(crate) fn parse_engine_recipe_json(value: Option) -> Result Result { + require_chat_task(request.task)?; let device_policy = normalize_vllm_device_policy(request.device_policy)?; let engine_recipe = accepted_engine_recipe(request.engine_recipe)?; let requested_gpu_indices = @@ -1009,6 +1024,7 @@ mod tests { device_policy: Some(DevicePolicy::GpuRequired), recipe_override: None, engine_recipe: None, + task: rocm_engine_protocol::ModelTask::Chat, })?; assert!( @@ -1660,12 +1676,50 @@ WARNING Failed to import from vllm._C with ImportError('libtorch_cuda.so: cannot device_policy: Some(DevicePolicy::GpuRequired), recipe_override: None, engine_recipe: Some(hint.clone()), + task: rocm_engine_protocol::ModelTask::Chat, })?; assert_eq!(response.engine_recipe, Some(hint)); Ok(()) } #[test] + fn resolve_and_launch_refuse_a_speech_to_text_model() { + let error = resolve_model_response(ResolveModelRequest { + model_ref: "Whisper-Tiny".to_owned(), + runtime_id: None, + device_policy: Some(DevicePolicy::GpuRequired), + recipe_override: None, + engine_recipe: None, + task: rocm_engine_protocol::ModelTask::Transcription, + }) + .expect_err("vLLM must not resolve a speech-to-text model"); + assert!( + error + .to_string() + .contains("serves chat models only; it cannot serve a speech-to-text model"), + "{error}" + ); + + let error = launch_service(LaunchRequest { + service_id: "svc".to_owned(), + env_id: None, + runtime_id: None, + model_ref: "Whisper-Tiny".to_owned(), + host: "127.0.0.1".to_owned(), + port: 0, + device_policy: Some(DevicePolicy::GpuRequired), + endpoint_mode: None, + engine_recipe: None, + gpu_selection: None, + task: rocm_engine_protocol::ModelTask::Transcription, + }) + .expect_err("vLLM must not launch a speech-to-text model"); + assert!( + error.to_string().contains("serves chat models only"), + "{error}" + ); + } + #[test] fn resolve_model_rejects_mismatched_engine_recipe() { let error = resolve_model_response(ResolveModelRequest { model_ref: "facebook/opt-125m".to_owned(), @@ -1676,6 +1730,7 @@ WARNING Failed to import from vllm._C with ImportError('libtorch_cuda.so: cannot "lemonade", ENGINE_RECIPE_CONTRACT_VERSION, )), + task: rocm_engine_protocol::ModelTask::Chat, }) .expect_err("mismatched engine recipe should fail"); @@ -1689,6 +1744,7 @@ WARNING Failed to import from vllm._C with ImportError('libtorch_cuda.so: cannot device_policy: Some(DevicePolicy::GpuRequired), recipe_override: None, engine_recipe: Some(test_engine_recipe(ENGINE_NAME, "999.0.0")), + task: rocm_engine_protocol::ModelTask::Chat, }) .expect_err("unsupported recipe contract should fail"); diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index 6ad87f9379..d5fcee6333 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -84,12 +84,7 @@ pub(crate) fn endpoint_response(request: EndpointRequest) -> Result bool { rocm_core::engine_state_inference_verified( + rocm_core::ModelTask::Chat, state_path, state, endpoint_url, diff --git a/tests/e2e-cucumber/features/bench.feature b/tests/e2e-cucumber/features/bench.feature index cbf66f5a32..ac9872d339 100644 --- a/tests/e2e-cucumber/features/bench.feature +++ b/tests/e2e-cucumber/features/bench.feature @@ -71,3 +71,12 @@ Feature: Benchmarking a served endpoint And a model is being served on GPU When the user benchmarks the served endpoint Then the benchmark reports measured throughput + + # `rocm bench load` drives chat completions. Pointed at a speech-to-text model + # it must refuse up front with the reason, not report a sweep of failures. + @id:bench-load-refuses-a-speech-to-text-model + Scenario: bench-07 - Benchmarking a speech-to-text model is refused with the reason + Given a speech-to-text model is being served + When the user benchmarks the served endpoint + Then the benchmark is refused because the model is a speech-to-text model + And no chat request reached the speech-to-text server diff --git a/tests/e2e-cucumber/features/chat.feature b/tests/e2e-cucumber/features/chat.feature index 0c08bba008..9c1af42296 100644 --- a/tests/e2e-cucumber/features/chat.feature +++ b/tests/e2e-cucumber/features/chat.feature @@ -107,3 +107,28 @@ Feature: Chat and endpoint detection When the user pipes a one-shot chat prompt through the CLI Then the CLI prints the assistant's reply And the model receives the piped prompt with its whitespace intact + + # A speech-to-text server cannot answer chat. Naming it as the chat model + # must fail with the reason, before any chat request is sent to it. + @id:chat-refuses-a-speech-to-text-model + Scenario: chat-10 - Chat refuses a speech-to-text model and says why + Given a speech-to-text model is being served and registered with the CLI + When the user asks the chat CLI to use the speech-to-text model + Then chat is refused because the model is a speech-to-text model + And no chat request reached the speech-to-text server + + # The interactive chat pane picks the newest ready local service. A newer + # speech-to-text server must not win that pick: the pane has to offer the + # chat model's endpoint, and the chat must reach only the chat model. + @id:chat-pane-skips-a-speech-to-text-service @requires-os:linux + Scenario: chat-11 - Interactive chat ignores a newer speech-to-text service + Given a running managed model is available locally + And a newer speech-to-text service is also running + When the user opens interactive chat + Then the local endpoint is shown for confirmation + When the user accepts the local endpoint + And the user sends a message to the managed model + Then the managed model's response is displayed + And no chat request reached the speech-to-text server + When the user quits interactive chat + Then interactive chat exits successfully diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 90caa1aee5..169c7cef2a 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -43,6 +43,7 @@ Feature: Model serving When the user sends a chat completion request Then the response contains a model reply And the response identifies the correct model + And the API docs link printed when it started opens the server's chat API docs # Large-model coverage (dogfooding W9): serve a representative large model for # each GPU platform end-to-end at least once. The model follows the lane's serve @@ -69,6 +70,7 @@ Feature: Model serving When the user sends a chat completion request Then the response contains a model reply And the response identifies the correct model + And the API docs link printed when it started opens the server's chat API docs # HF-checkpoint direct-serve canary (EAI-8026). An `owner/repo:variant` ref # bypasses Lemonade's model router and runs a packaged llama-server directly @@ -341,3 +343,37 @@ Feature: Model serving Scenario: serve-25 - The recommended model list prints ASCII-only text When the user lists recommended models Then the model list output contains only ASCII characters + + # Speech-to-text models (Whisper) serve only through Lemonade; vLLM advertises + # chat only. The refusal is decided by the engine's own advertised tasks + # before the GPU gate, so it gates every PR (ungated) and must not be masked + # by the no-GPU message on the mock lane. The command it recommends is the + # one serve-27 runs on a GPU lane. + @id:serve-speech-to-text-model-refused-by-chat-only-engine + Scenario: serve-26 - A speech-to-text model sent to a chat-only engine is refused with a working alternative + When the user serves a speech-to-text model on an engine that only serves chat + Then serving is refused because that engine cannot serve speech-to-text models + And the user is told the command that serves it with Lemonade + + # The real thing: Lemonade's whisper.cpp backend on the GPU (ROCm, or Vulkan + # where Lemonade reports ROCm unsupported; its CPU build is never selected). + # Runs on the Lemonade + # GPU lanes. Whisper-Tiny keeps the download under 100 MB. + @id:serve-speech-to-text-model-prints-its-transcription-endpoint @requires-gpu @requires-engine:lemonade + Scenario: serve-27 - A served speech-to-text model answers on the endpoint the CLI prints + Given a managed runtime is active + When the user serves a speech-to-text model with the engine the CLI recommended + Then the CLI prints the speech-to-text endpoint of the served model + And a recording sent to that endpoint is transcribed + And the running service is listed as speech-to-text + And the API docs link printed when it started opens the server's speech-to-text API docs + + # A chat client appends /chat/completions to a listed base URL, which a + # speech-to-text server does not have; the list must also name the route it + # does serve, and that URL must reach the registered server's route. + @id:serve-speech-to-text-server-listed-with-its-task + Scenario: serve-28 - A running speech-to-text server is listed with its task and the endpoint it answers on + Given a speech-to-text model is being served and registered with the CLI + When the user lists running services + Then the service is listed as speech-to-text with its transcription endpoint + And the listed speech-to-text endpoint reaches that server's transcription route diff --git a/tests/e2e-cucumber/features/service_docs.feature b/tests/e2e-cucumber/features/service_docs.feature new file mode 100644 index 0000000000..c1392bca69 --- /dev/null +++ b/tests/e2e-cucumber/features/service_docs.feature @@ -0,0 +1,59 @@ +Feature: API docs for running model servers + + Every running model server gets API docs with a working "Try it out", + served on loopback by the background helper `rocm serve` starts. The CLI + prints a server's docs link only once those docs answer. + + # @serial: each scenario starts a real background helper. Run alongside the + # rest of the mock lane, these four pushed the dash throughput-window + # scenarios (dash-08/09) past their timing window in full runs; run alone + # they add a few seconds and no concurrent load. + + @id:service-docs-link-opens-chat-routes @serial + Scenario: service-docs-01 - A running server's API docs are linked and list its chat routes + Given a model is being served + And the model is registered with the CLI + And the background helper is running + When the user checks for running services + Then the running server has an API docs link + And the link opens API docs listing the chat routes with the server's model + + @id:service-docs-try-it-reaches-server @serial + Scenario: service-docs-02 - Trying a route from the API docs reaches the model server + Given a model is being served + And the model is registered with the CLI + And the background helper is running + When the user tries the chat completions route from the server's API docs + Then the model server answers the request + + # `rocm services stop` signals the processes the record names, so the record + # is pointed at a disposable process first (`sleep`, hence Linux only). + @id:service-docs-gone-after-stop @requires-os:linux @serial + Scenario: service-docs-03 - A stopped server's API docs say it is not running + Given a running managed model is available locally + And the managed model's record points at a disposable process + And the background helper is running + And the user has opened the server's API docs + When the user stops the managed model + Then the server's API docs say it is not running + And the services list still shows the stopped server, without an API docs link + + # The CLI prints a link only once the docs answer for that server. The link + # `rocm serve` prints is checked on the GPU serve canaries serve-05 (vLLM) and + # serve-07 (Lemonade; an expected failure on Linux Lemonade lanes for an + # unrelated reason, so effective on Windows Lemonade lanes only). + @id:service-docs-no-link-without-helper @serial + Scenario: service-docs-04 - Without the background helper no API docs link is printed + Given a model is being served + And the model is registered with the CLI + And the background helper is not running + When the user checks for running services + Then the running server has no API docs link + + @id:service-docs-speech-to-text-upload @serial + Scenario: service-docs-05 - A speech-to-text server's API docs offer an audio upload instead of chat + Given a speech-to-text model is being served and registered with the CLI + And the background helper is running + When the user opens the server's API docs from the services list + Then the API docs list the transcription route and no chat routes + And a recording uploaded through the API docs is transcribed diff --git a/tests/e2e-cucumber/src/mock_server.rs b/tests/e2e-cucumber/src/mock_server.rs index 74170ce737..35a80913b3 100644 --- a/tests/e2e-cucumber/src/mock_server.rs +++ b/tests/e2e-cucumber/src/mock_server.rs @@ -433,6 +433,9 @@ vllm:time_per_output_token_seconds_count{{model=\"mock\"}} {tpot_count} } } +/// The fixed transcript [`MockServer::start_transcription`] answers with. +pub const MOCK_TRANSCRIPT: &str = "mock transcript of the uploaded audio"; + #[derive(Debug)] pub struct MockServer { server: ServerHandle, @@ -442,6 +445,10 @@ pub struct MockServer { chat_paths: Arc>>, /// `Some` only when started with [`MockServer::start_with_scripted_metrics`]. scripted: Option>, + /// Every URI path a speech-to-text request arrived on. Only + /// [`MockServer::start_transcription`] serves that route; every other + /// constructor leaves this empty. + transcription_paths: Arc>>, } impl MockServer { @@ -487,6 +494,7 @@ impl MockServer { last_chat_request, scripted: Some(scripted), chat_paths, + transcription_paths: Arc::new(Mutex::new(Vec::new())), } } @@ -542,6 +550,7 @@ impl MockServer { scripted: None, last_chat_request, chat_paths, + transcription_paths: Arc::new(Mutex::new(Vec::new())), } } @@ -580,9 +589,100 @@ impl MockServer { last_chat_request: Arc::new(Mutex::new(None)), chat_paths: Arc::new(Mutex::new(Vec::new())), scripted: None, + transcription_paths: Arc::new(Mutex::new(Vec::new())), } } + /// A speech-to-text server, shaped like Lemonade serving a Whisper model: + /// it lists the model, answers `POST /v1/audio/transcriptions` with a fixed + /// transcript, and has no chat route — a chat request is refused with `404`, + /// but its path is still recorded so a scenario can prove the CLI never + /// sent one. + pub fn start_transcription(model_name: &str) -> Self { + #[derive(Clone)] + struct TranscriptionState { + model_name: String, + chat_paths: Arc>>, + transcription_paths: Arc>>, + } + async fn models(State(state): State) -> Json { + Json(json!({ + "object": "list", + "data": [{"id": state.model_name, "object": "model"}] + })) + } + /// Answers only what a real transcription server would: a multipart + /// upload with a non-empty `file` part and this server's `model`. + /// Anything else is a 400, so a request mangled on its way here (a + /// lost boundary, a dropped body, the wrong model) cannot pass for a + /// transcription. + async fn transcribe( + State(state): State, + uri: axum::http::Uri, + headers: axum::http::HeaderMap, + body: axum::body::Bytes, + ) -> (StatusCode, Json) { + state + .transcription_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(uri.path().to_string()); + match transcription_upload_problem(&headers, &body, &state.model_name) { + None => (StatusCode::OK, Json(json!({"text": MOCK_TRANSCRIPT}))), + Some(problem) => ( + StatusCode::BAD_REQUEST, + Json(json!({"error": {"message": problem}})), + ), + } + } + async fn refuse_chat( + State(state): State, + uri: axum::http::Uri, + ) -> StatusCode { + state + .chat_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(uri.path().to_string()); + StatusCode::NOT_FOUND + } + + let chat_paths = Arc::new(Mutex::new(Vec::new())); + let transcription_paths = Arc::new(Mutex::new(Vec::new())); + let state = TranscriptionState { + model_name: model_name.to_string(), + chat_paths: Arc::clone(&chat_paths), + transcription_paths: Arc::clone(&transcription_paths), + }; + let app = Router::new() + .route("/v1/models", get(models)) + .route("/models", get(models)) + .route("/v1/audio/transcriptions", post(transcribe)) + .route("/v1/chat/completions", post(refuse_chat)) + .route("/chat/completions", post(refuse_chat)) + .with_state(state); + Self { + // Its own thread and runtime, not the harness's: steps block the + // harness's workers on CLI subprocesses, and with many scenarios + // running at once a request to this server could sit unserved + // until the client gave up. + server: http_server::spawn_on_own_thread(app), + last_chat_request: Arc::new(Mutex::new(None)), + chat_paths, + scripted: None, + transcription_paths, + } + } + + /// Every URI path a speech-to-text request has arrived on, in order. + #[must_use] + pub fn transcription_paths(&self) -> Vec { + self.transcription_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + /// The OpenAI-compatible API root (`.../v1`) the CLI is pointed at. pub fn base_url(&self) -> String { self.server @@ -704,6 +804,19 @@ pub struct ServiceRecordOptions { /// refused before anything is stopped, which makes a failed restart /// observable without an engine. pub host: Option<&'static str>, + /// What the service does (`"transcription"` for a speech-to-text server). + /// `None` (the default) writes no `task` key at all — a record from before + /// services carried one, which the CLI reads as a chat service. An explicit + /// `null` would not do: the CLI would fail to parse the record and skip it. + pub task: Option<&'static str>, + /// The engine the record names. `None` (the default) keeps `vllm`. + pub engine: Option<&'static str>, + /// The record's service id, which also names its manifest file. `None` (the + /// default) keeps `e2e-mock`; set it to plant a second service beside it. + pub service_id: Option<&'static str>, + /// When the service was created. `None` (the default) keeps the fixed + /// historical value; set a later one to make a planted service the newest. + pub created_at_unix_ms: Option, } impl Default for ServiceRecordOptions { @@ -716,6 +829,10 @@ impl Default for ServiceRecordOptions { supervisor_start_ticks: None, runtime_id: None, host: None, + task: None, + engine: None, + service_id: None, + created_at_unix_ms: None, } } } @@ -732,6 +849,49 @@ pub fn write_service_record(services_dir: &Path, model: &str, port: u16) { write_service_record_with(services_dir, model, port, ServiceRecordOptions::default()); } +/// Why a transcription request is not a valid upload for `model`, or `None` +/// when it is. +fn transcription_upload_problem( + headers: &axum::http::HeaderMap, + body: &[u8], + model: &str, +) -> Option { + let content_type = headers + .get(axum::http::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default(); + let Some(boundary) = content_type + .strip_prefix("multipart/form-data") + .and_then(|rest| rest.split("boundary=").nth(1)) + .map(|boundary| boundary.trim_matches('"')) + .filter(|boundary| !boundary.is_empty()) + else { + return Some(format!("expected a multipart upload, got `{content_type}`")); + }; + let body = String::from_utf8_lossy(body); + let mut file_bytes = None; + let mut sent_model = None; + for part in body.split(&format!("--{boundary}")) { + let Some((part_headers, content)) = part.split_once("\r\n\r\n") else { + continue; + }; + let content = content.strip_suffix("\r\n").unwrap_or(content); + if part_headers.contains("name=\"file\"") { + file_bytes = Some(content.len()); + } else if part_headers.contains("name=\"model\"") { + sent_model = Some(content.to_owned()); + } + } + match (file_bytes, sent_model) { + (None | Some(0), _) => Some("the upload has no audio in its `file` part".to_owned()), + (_, None) => Some("the upload names no `model`".to_owned()), + (_, Some(sent)) if sent != model => { + Some(format!("this server serves `{model}`, not `{sent}`")) + } + _ => None, + } +} + /// Like [`write_service_record`], but with caller-specified lifecycle fields. /// /// See [`ServiceRecordOptions`] -- e.g. a "still loading" status/startup_phase, @@ -749,9 +909,10 @@ pub fn write_service_record_with( // The CLI only extracts host:port from `endpoint_url` and appends // `/v1/models` for its readiness probe, which the mock serves. - let record = json!({ - "service_id": "e2e-mock", - "engine": "vllm", + let service_id = options.service_id.unwrap_or("e2e-mock"); + let mut record = json!({ + "service_id": service_id, + "engine": options.engine.unwrap_or("vllm"), "model_ref": model, "canonical_model_id": model, "host": options.host.unwrap_or("127.0.0.1"), @@ -773,13 +934,16 @@ pub fn write_service_record_with( "engine_recipe_json": null, "restart_count": 0, "last_restart_unix_ms": null, - "manifest_path": services_dir.join("e2e-mock.json"), - "log_path": services_dir.join("e2e-mock.log"), - "engine_state_path": services_dir.join("e2e-mock.state.json"), - "created_at_unix_ms": 1_700_000_000_000_u64, + "manifest_path": services_dir.join(format!("{service_id}.json")), + "log_path": services_dir.join(format!("{service_id}.log")), + "engine_state_path": services_dir.join(format!("{service_id}.state.json")), + "created_at_unix_ms": options.created_at_unix_ms.unwrap_or(1_700_000_000_000), }); + if let Some(task) = options.task { + record["task"] = json!(task); + } std::fs::write( - services_dir.join("e2e-mock.json"), + services_dir.join(format!("{service_id}.json")), serde_json::to_vec_pretty(&record).expect("failed to serialize record"), ) .expect("failed to write service record"); diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index cab0f3befb..115bc45385 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -37,9 +37,11 @@ mod e2e { pub mod runtime_lifecycle_steps; pub mod runtime_steps; pub mod service_cleanup_steps; + pub mod service_docs_steps; pub mod service_stop_steps; pub mod serving_steps; pub mod skill_steps; + pub mod speech_steps; pub mod storage_steps; pub mod therock_steps; pub mod tui_driver; @@ -52,6 +54,9 @@ mod e2e { #[derive(Debug, cucumber::World)] pub struct E2eWorld { pub mock: Option, + /// A speech-to-text server planted beside `mock`, for scenarios that need + /// both a chat and a speech-to-text service running at once. + pub speech_mock: Option, /// Loopback file server used by artifact-prefetch scenarios. Kept on the /// World so it remains alive while the real `rocmd` subprocess downloads. pub artifact_server: Option, @@ -170,6 +175,14 @@ pub struct E2eWorld { /// refusal envelope (it carries a `refused` key). Recorded when the preview /// is read, so a later step can check the card against what the CLI said. pub dash_report_refused: Option, + /// A `rocm daemon` a scenario started so its API docs answer; killed at + /// teardown, before the isolated tree it reads is removed. + pub background_helper: Option, + /// The API docs link a `service-docs` scenario read from the CLI. + pub docs_link: Option, + /// What a successful real `rocm serve` printed, for steps that check the + /// launch output after the server came up. + pub serve_stdout: Option, } /// One scenario's resolved expectation plus the identity needed to report it. @@ -294,6 +307,7 @@ impl Default for E2eWorld { Self { mock: None, + speech_mock: None, artifact_server: None, paced_download_server: None, comfyui_port_listener: None, @@ -323,6 +337,9 @@ impl Default for E2eWorld { skill_symptom: None, recorded_process: None, dash_report_refused: None, + background_helper: None, + docs_link: None, + serve_stdout: None, } } } @@ -653,9 +670,16 @@ impl Drop for E2eWorld { let _ = process.kill(); let _ = process.wait(); } + if let Some(mut helper) = self.background_helper.take() { + let _ = helper.kill(); + let _ = helper.wait(); + } if let Some(mock) = self.mock.take() { mock.stop(); } + if let Some(mock) = self.speech_mock.take() { + mock.stop(); + } self.artifact_server.take(); self.paced_download_server.take(); // A scenario that ran `rocm serve --managed` left a DETACHED supervisor + diff --git a/tests/e2e-cucumber/tests/e2e/service_docs_steps.rs b/tests/e2e-cucumber/tests/e2e/service_docs_steps.rs new file mode 100644 index 0000000000..ebe9916cbc --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/service_docs_steps.rs @@ -0,0 +1,391 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `service_docs.feature`: the API docs the background helper serves +//! for every running model server. +//! +//! The helper is the real `rocm daemon`, started in the scenario's isolated +//! tree on a port of its own (set in that tree's `config.json`), so parallel +//! scenarios and a helper already running on the host never share docs. + +use std::time::{Duration, Instant}; + +use cucumber::{given, then, when}; +use serde_json::Value; + +use crate::E2eWorld; + +/// How long a cold `rocm daemon` gets to start answering. +const HELPER_START_TIMEOUT: Duration = Duration::from_secs(30); + +fn free_port() -> u16 { + std::net::TcpListener::bind(("127.0.0.1", 0)) + .and_then(|listener| listener.local_addr()) + .map(|addr| addr.port()) + .expect("failed to reserve a loopback port") +} + +/// The docs link `rocm services list` printed for the one listed server. +fn docs_link_in(listing: &str) -> Option { + listing + .lines() + .find_map(|line| line.trim().strip_prefix("docs: ")) + .map(str::to_owned) +} + +async fn get(url: &str) -> reqwest::Response { + reqwest::get(url) + .await + .unwrap_or_else(|error| panic!("GET {url} failed: {error}")) +} + +/// The OpenAPI description a docs page loads, found the way the page finds it: +/// from the Swagger UI initializer served next to the page. +async fn openapi_behind(docs_link: &str) -> (String, Value) { + let initializer = get(&format!("{docs_link}swagger-initializer.js")) + .await + .text() + .await + .expect("docs initializer body"); + let start = initializer + .find("/services/") + .unwrap_or_else(|| panic!("the docs page names no description:\n{initializer}")); + let path: String = initializer[start..] + .chars() + .take_while(|character| *character != '"') + .collect(); + let origin = docs_link + .split("/services/") + .next() + .expect("docs link has an origin"); + let openapi = get(&format!("{origin}{path}")) + .await + .json::() + .await + .expect("the docs description is JSON"); + (origin.to_owned(), openapi) +} + +/// Point this scenario's API docs at a free port of their own, in the isolated +/// `config.json`, and return it. A helper `rocm serve` leaves behind in another +/// scenario keeps its own port, so it can never answer for this one. Calling it +/// again keeps the port already chosen: a serve retried within one scenario +/// reuses the helper the first attempt started, which listens there. +pub(crate) fn give_api_docs_a_port_of_their_own(world: &E2eWorld) -> u16 { + let paths = world.app_paths(); + std::fs::create_dir_all(&paths.config_dir).expect("failed to create config dir"); + let config_path = paths.config_path(); + let mut config: Value = std::fs::read(&config_path) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .unwrap_or_else(|| serde_json::json!({})); + if let Some(port) = config["api_docs"]["port"] + .as_u64() + .and_then(|port| u16::try_from(port).ok()) + { + return port; + } + let port = free_port(); + config["api_docs"] = serde_json::json!({ "enabled": true, "port": port }); + std::fs::write(&config_path, serde_json::to_vec_pretty(&config).unwrap()) + .expect("failed to write config.json"); + port +} + +/// The docs link in `rocm serve`'s output: the deployment summary's `docs` row +/// or the plain output's `docs:` line. +fn docs_link_in_serve_output(stdout: &str) -> Option { + stdout.lines().find_map(|line| { + let rest = line.trim().strip_prefix("docs")?; + let link = rest.trim_start_matches(':').trim(); + link.starts_with("http://").then(|| link.to_owned()) + }) +} + +// ── Given ────────────────────────────────────────────────────────── + +#[given("the background helper is not running")] +async fn background_helper_not_running(world: &mut E2eWorld) { + // Nothing listens on this scenario's docs port: the state a host is in + // when no helper is up, which a helper running on the host itself cannot + // disturb. + give_api_docs_a_port_of_their_own(world); +} + +#[given("the background helper is running")] +async fn background_helper_running(world: &mut E2eWorld) { + let port = give_api_docs_a_port_of_their_own(world); + let paths = world.app_paths(); + + // The helper's own output names why docs did not come up (`api_docs: off + // (...)`), so it is kept for the failure message rather than discarded. + let log_path = paths.data_dir.join("background-helper.log"); + std::fs::create_dir_all(&paths.data_dir).expect("failed to create data dir"); + let log = std::fs::File::create(&log_path).expect("failed to create helper log"); + let mut command = std::process::Command::new(crate::rocm_binary()); + command + .arg("daemon") + .stdin(std::process::Stdio::null()) + .stdout(log.try_clone().expect("failed to share helper log")) + .stderr(log); + world.isolate_cmd(&mut command); + world.background_helper = Some(command.spawn().expect("failed to start rocm daemon")); + + let health = format!("http://127.0.0.1:{port}/health"); + let deadline = Instant::now() + HELPER_START_TIMEOUT; + while !reqwest::get(&health) + .await + .is_ok_and(|response| response.status().is_success()) + { + let exited = world + .background_helper + .as_mut() + .and_then(|helper| helper.try_wait().ok().flatten()); + let output = || std::fs::read_to_string(&log_path).unwrap_or_default(); + assert!( + exited.is_none(), + "the background helper exited ({exited:?}) before its API docs answered:\n{}", + output() + ); + assert!( + Instant::now() < deadline, + "the background helper's API docs did not answer on port {port} within \ + {HELPER_START_TIMEOUT:?}; its output:\n{}", + output() + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +#[given("the user has opened the server's API docs")] +async fn user_opened_docs(world: &mut E2eWorld) { + let listing = crate::run_rocm_ok(world, &["services", "list"]); + let link = docs_link_in(&listing) + .unwrap_or_else(|| panic!("premise: a running server has a docs link:\n{listing}")); + let page = get(&link).await; + assert!(page.status().is_success(), "premise: the docs open"); + world.docs_link = Some(link); +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("the user tries the chat completions route from the server's API docs")] +async fn user_tries_chat_route(world: &mut E2eWorld) { + let listing = crate::run_rocm_ok(world, &["services", "list"]); + let link = docs_link_in(&listing) + .unwrap_or_else(|| panic!("no API docs link in the services list:\n{listing}")); + let (origin, openapi) = openapi_behind(&link).await; + // What "Try it out" sends: the route's example body, to the server the + // description names. + let server = openapi["servers"][0]["url"] + .as_str() + .expect("the docs name a server to send requests to"); + let body = openapi["paths"]["/v1/chat/completions"]["post"]["requestBody"]["content"] + ["application/json"]["example"] + .clone(); + let response = reqwest::Client::new() + .post(format!("{origin}{server}/v1/chat/completions")) + .json(&body) + .send() + .await + .expect("try-it request failed to send"); + let status = response.status(); + let answer = response.text().await.unwrap_or_default(); + world.cli_output = Some(format!("{status}\n{answer}")); +} + +#[when("the user opens the server's API docs from the services list")] +async fn user_opens_docs_from_list(world: &mut E2eWorld) { + let listing = crate::run_rocm_ok(world, &["services", "list"]); + let link = docs_link_in(&listing) + .unwrap_or_else(|| panic!("no API docs link in the services list:\n{listing}")); + let page = get(&link).await; + assert!(page.status().is_success(), "the docs page did not open"); + world.docs_link = Some(link); +} + +#[when("the user stops the managed model")] +async fn user_stops_managed_model(world: &mut E2eWorld) { + crate::run_rocm_ok(world, &["services", "stop", "e2e-mock", "--yes"]); +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then("the running server has an API docs link")] +async fn running_server_has_docs_link(world: &mut E2eWorld) { + let listing = world.cli_output.as_deref().expect("no services listing"); + let link = docs_link_in(listing) + .unwrap_or_else(|| panic!("no API docs link in the services list:\n{listing}")); + assert!( + link.ends_with("/services/e2e-mock/docs/"), + "the link names a different server: {link}" + ); + world.docs_link = Some(link); +} + +#[then("the link opens API docs listing the chat routes with the server's model")] +async fn link_opens_chat_docs(world: &mut E2eWorld) { + let link = world.docs_link.clone().expect("no docs link"); + let page = get(&link).await; + assert!(page.status().is_success(), "the docs page did not open"); + assert!( + page.text().await.unwrap_or_default().contains("swagger-ui"), + "the link did not open the API docs page" + ); + let (_, openapi) = openapi_behind(&link).await; + for route in ["/v1/models", "/v1/chat/completions", "/v1/completions"] { + assert!( + openapi["paths"].get(route).is_some(), + "the docs do not list {route}:\n{openapi:#}" + ); + } + let model = world.model_name.as_deref().expect("no model name"); + assert_eq!( + openapi["paths"]["/v1/chat/completions"]["post"]["requestBody"]["content"]["application/json"] + ["example"]["model"], + model, + "the docs' example does not use the server's model" + ); +} + +#[then("the model server answers the request")] +async fn model_server_answers(world: &mut E2eWorld) { + let answer = world.cli_output.as_deref().expect("no try-it answer"); + let (status, body) = answer.split_once('\n').unwrap_or((answer, "")); + assert!( + status.starts_with("200"), + "try-it was not answered: {answer}" + ); + let body: Value = serde_json::from_str(body) + .unwrap_or_else(|_| panic!("the answer is not the server's JSON: {answer}")); + assert!( + body["choices"][0]["message"]["content"].is_string(), + "the answer is not a chat completion: {body:#}" + ); +} + +#[then("the server's API docs say it is not running")] +async fn docs_say_not_running(world: &mut E2eWorld) { + let link = world.docs_link.clone().expect("no docs link"); + let page = get(&link).await; + assert_eq!(page.status(), reqwest::StatusCode::NOT_FOUND); + let text = page.text().await.unwrap_or_default(); + assert!(text.contains("is not running"), "{text}"); +} + +#[then("the services list still shows the stopped server, without an API docs link")] +async fn stopped_server_listed_without_docs_link(world: &mut E2eWorld) { + // `--all`: the default list hides stopped servers, so it could not show a + // docs line for one whatever the CLI did. + let listing = crate::run_rocm_ok(world, &["services", "list", "--all"]); + assert!( + listing.contains("- e2e-mock") && listing.contains("status: stopped"), + "premise: the stopped server is still listed:\n{listing}" + ); + assert_eq!(docs_link_in(&listing), None, "{listing}"); +} + +#[then("the running server has no API docs link")] +async fn running_server_has_no_docs_link(world: &mut E2eWorld) { + let listing = world.cli_output.as_deref().expect("no services listing"); + assert!( + listing.contains("- e2e-mock"), + "premise: the running server is listed:\n{listing}" + ); + assert_eq!(docs_link_in(listing), None, "{listing}"); +} + +/// Opens the docs link `rocm serve` printed, checks the docs list exactly +/// `routes` besides the health route, and lists the models through the docs +/// page so "Try it out" is proven to reach the real engine. +async fn serve_printed_docs_link_opens(stdout: &str, routes: &[&str], absent: &[&str]) { + let link = docs_link_in_serve_output(stdout) + .unwrap_or_else(|| panic!("`rocm serve` printed no API docs link:\n{stdout}")); + let (origin, openapi) = openapi_behind(&link).await; + for route in routes { + assert!( + openapi["paths"].get(*route).is_some(), + "the docs do not list {route}:\n{openapi:#}" + ); + } + for route in absent { + assert!( + openapi["paths"].get(*route).is_none(), + "the docs list {route}, which this server does not serve:\n{openapi:#}" + ); + } + let server = openapi["servers"][0]["url"] + .as_str() + .expect("the docs name a server to send requests to"); + let models = get(&format!("{origin}{server}/v1/models")).await; + assert!( + models.status().is_success(), + "try-it through the docs did not reach the engine: {}", + models.status() + ); +} + +#[then("the API docs link printed when it started opens the server's chat API docs")] +async fn serve_printed_chat_docs_link_opens(world: &mut E2eWorld) { + let stdout = world.serve_stdout.clone().expect("no serve output"); + serve_printed_docs_link_opens( + &stdout, + &["/v1/models", "/v1/chat/completions", "/v1/completions"], + &["/v1/audio/transcriptions"], + ) + .await; +} + +#[then("the API docs link printed when it started opens the server's speech-to-text API docs")] +async fn serve_printed_speech_docs_link_opens(world: &mut E2eWorld) { + let stdout = world.serve_stdout.clone().expect("no serve output"); + serve_printed_docs_link_opens( + &stdout, + &["/v1/models", "/v1/audio/transcriptions"], + &["/v1/chat/completions", "/v1/completions"], + ) + .await; +} + +#[then("the API docs list the transcription route and no chat routes")] +async fn docs_list_transcription_only(world: &mut E2eWorld) { + let link = world.docs_link.clone().expect("no docs link"); + let (_, openapi) = openapi_behind(&link).await; + let upload = &openapi["paths"]["/v1/audio/transcriptions"]["post"]["requestBody"]["content"]["multipart/form-data"] + ["schema"]["properties"]; + assert_eq!( + upload["file"]["format"], "binary", + "the transcription route offers no audio upload:\n{openapi:#}" + ); + for chat_route in ["/v1/chat/completions", "/v1/completions"] { + assert!( + openapi["paths"].get(chat_route).is_none(), + "a speech-to-text server's docs list {chat_route}:\n{openapi:#}" + ); + } +} + +#[then("a recording uploaded through the API docs is transcribed")] +async fn recording_uploaded_through_docs(world: &mut E2eWorld) { + let link = world.docs_link.clone().expect("no docs link"); + let (origin, openapi) = openapi_behind(&link).await; + let server = openapi["servers"][0]["url"] + .as_str() + .expect("the docs name a server to send requests to"); + let model = openapi["paths"]["/v1/audio/transcriptions"]["post"]["requestBody"]["content"] + ["multipart/form-data"]["schema"]["properties"]["model"]["default"] + .as_str() + .expect("the upload form fills in the model") + .to_owned(); + let (status, body) = crate::e2e::speech_steps::transcribe( + &format!("{origin}{server}/v1/audio/transcriptions"), + &model, + ) + .await; + assert_eq!(status, 200, "the upload through the docs failed: {body}"); + assert!( + body.get("text").is_some_and(Value::is_string), + "the answer is not a transcript: {body}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 7ce18b326b..6224c744c9 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -103,7 +103,7 @@ fn stall_headline(invocation: &str, ready_substr: &str, timeout_secs: u64) -> St /// /// Mirrors what [`setup_gpu_model`] has always collected, so a stalled serve /// reports the same evidence wherever it happens. -fn serve_failure_evidence( +pub(crate) fn serve_failure_evidence( world: &E2eWorld, headline: &str, device_state: &str, @@ -145,10 +145,12 @@ fn serve_failure_evidence( async fn serve_and_wait(world: &mut E2eWorld, args: &[&str], model: &str, ready_substr: &str) { let timeout_secs = serve_timeout_for(world); let device_state = ensure_serve_port_free().await; + crate::e2e::service_docs_steps::give_api_docs_a_port_of_their_own(world); let (stdout, stderr, rc) = crate::run_rocm(world, args); if rc == 0 && model_is_ready(&MODELS_URL, Some(ready_substr), timeout_secs).await { world.endpoint = Some(SERVE_BASE_URL.to_string()); world.model_name = Some(model.to_string()); + world.serve_stdout = Some(stdout); return; } let invocation = format!("`rocm {}`", args.join(" ")); @@ -187,7 +189,7 @@ const ASSISTANT_PORT: u16 = 8001; /// (see [`wait_for_free_vram`]). Callers that only need the reset can ignore it; /// a serve that then fails to become ready reports it, because "the previous /// engine had not released the GPU yet" is otherwise invisible in the log. -async fn ensure_serve_port_free() -> String { +pub(crate) async fn ensure_serve_port_free() -> String { // Always kill any listener on the shared port — NOT just one that already // answers /v1/models. A prior scenario's vLLM that is still LOADING holds the // port and GPU memory without yet serving /v1/models; if we only checked HTTP @@ -544,11 +546,13 @@ async fn serve_gpu_model(world: &mut E2eWorld) { for attempt in 1..=attempts { made = attempt; let device_state = ensure_serve_port_free().await; + crate::e2e::service_docs_steps::give_api_docs_a_port_of_their_own(world); let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", model, "--engine", engine, "--managed"]); if rc == 0 && model_is_ready(&MODELS_URL, Some(ready_substr), timeout_secs).await { world.endpoint = Some(SERVE_BASE_URL.to_string()); world.model_name = Some(model.to_string()); + world.serve_stdout = Some(stdout); return; } // `serve_failure_evidence` reads and copies out the log BEFORE it stops diff --git a/tests/e2e-cucumber/tests/e2e/speech_steps.rs b/tests/e2e-cucumber/tests/e2e/speech_steps.rs new file mode 100644 index 0000000000..60a6778d94 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/speech_steps.rs @@ -0,0 +1,440 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for serving speech-to-text models and for the commands that must not +//! mistake one for a chat model. + +use cucumber::{given, then, when}; +use e2e_cucumber::mock_server::{MOCK_TRANSCRIPT, MockServer, ServiceRecordOptions}; + +use crate::E2eWorld; + +/// The speech-to-text model the scenarios use: the smallest Whisper recipe, so +/// the GPU scenario's download stays small. +const SPEECH_MODEL_REF: &str = "whisper-tiny"; +const SPEECH_MODEL_ID: &str = "Whisper-Tiny"; + +fn cli_text(world: &E2eWorld) -> String { + format!( + "{}{}", + world.cli_output.as_deref().unwrap_or_default(), + world.cli_stderr.as_deref().unwrap_or_default() + ) +} + +fn record_cli(world: &mut E2eWorld, (stdout, stderr, rc): (String, String, i32)) { + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// One second of 16 kHz mono 16-bit silence, as a WAV file. +fn silent_wav() -> Vec { + let data_len: u32 = 32_000; + let mut wav = Vec::with_capacity(44 + data_len as usize); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(36 + data_len).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16_u32.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&16_000_u32.to_le_bytes()); + wav.extend_from_slice(&32_000_u32.to_le_bytes()); + wav.extend_from_slice(&2_u16.to_le_bytes()); + wav.extend_from_slice(&16_u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_len.to_le_bytes()); + wav.resize(44 + data_len as usize, 0); + wav +} + +/// POST a recording to an OpenAI-compatible transcription URL, the way an app +/// would, and return the HTTP status and parsed body. +/// +/// Sent from a blocking thread, not the harness's runtime: steps block that +/// runtime's workers on CLI subprocesses, and with a full lane of scenarios an +/// async request can wait out its whole timeout without being polled. +pub(crate) async fn transcribe(url: &str, model: &str) -> (u16, serde_json::Value) { + const BOUNDARY: &str = "e2e-speech-boundary"; + let mut body = format!( + "--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n{model}\r\n\ + --{BOUNDARY}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"clip.wav\"\r\n\ + Content-Type: audio/wav\r\n\r\n" + ) + .into_bytes(); + body.extend_from_slice(&silent_wav()); + body.extend_from_slice(format!("\r\n--{BOUNDARY}--\r\n").as_bytes()); + let content_type = format!("multipart/form-data; boundary={BOUNDARY}"); + let target = url.to_owned(); + let (status, text) = + tokio::task::spawn_blocking(move || blocking_post(&target, &content_type, &body)) + .await + .expect("the transcription request thread panicked") + .unwrap_or_else(|error| panic!("transcription request to {url} failed: {error}")); + let json = serde_json::from_str(&text).unwrap_or(serde_json::Value::String(text)); + (status, json) +} + +/// A plain HTTP/1.1 POST over a socket, with its own two-minute deadline. +/// Returns the status and the body, de-chunked when the server streamed it. +fn blocking_post(url: &str, content_type: &str, body: &[u8]) -> std::io::Result<(u16, String)> { + use std::io::{Read, Write}; + let invalid = |what: String| std::io::Error::new(std::io::ErrorKind::InvalidData, what); + let parsed = reqwest::Url::parse(url).map_err(|error| invalid(error.to_string()))?; + let host = parsed.host_str().unwrap_or("127.0.0.1").to_owned(); + let port = parsed.port_or_known_default().unwrap_or(80); + let deadline = std::time::Duration::from_mins(2); + let mut stream = std::net::TcpStream::connect((host.as_str(), port))?; + stream.set_read_timeout(Some(deadline))?; + stream.set_write_timeout(Some(deadline))?; + let path = match parsed.query() { + Some(query) => format!("{}?{query}", parsed.path()), + None => parsed.path().to_owned(), + }; + write!( + stream, + "POST {path} HTTP/1.1\r\nHost: {host}:{port}\r\nContent-Type: {content_type}\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + )?; + stream.write_all(body)?; + let mut raw = Vec::new(); + stream.read_to_end(&mut raw)?; + let text = String::from_utf8_lossy(&raw); + let (headers, payload) = text + .split_once("\r\n\r\n") + .ok_or_else(|| invalid(format!("no HTTP header terminator in: {text}")))?; + let status = headers + .split_whitespace() + .nth(1) + .and_then(|code| code.parse().ok()) + .ok_or_else(|| invalid(format!("no HTTP status in: {headers}")))?; + let chunked = headers + .lines() + .any(|line| line.to_ascii_lowercase().replace(' ', "") == "transfer-encoding:chunked"); + let body = if chunked { + dechunk(payload) + } else { + payload.to_owned() + }; + Ok((status, body)) +} + +/// The payload of a `Transfer-Encoding: chunked` body. +fn dechunk(mut payload: &str) -> String { + let mut body = String::new(); + while let Some((size, rest)) = payload.split_once("\r\n") { + let Ok(size) = usize::from_str_radix(size.trim(), 16) else { + break; + }; + if size == 0 || rest.len() < size { + break; + } + body.push_str(&rest[..size]); + payload = rest[size..].trim_start_matches("\r\n"); + } + body +} + +/// The URL on the output line `