diff --git a/Cargo.lock b/Cargo.lock index ed1f160cf..1e26e3f26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2118,6 +2118,8 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -3590,6 +3592,7 @@ dependencies = [ "base64 0.22.1", "bytes", "futures-core", + "futures-util", "http", "http-body", "http-body-util", @@ -3604,12 +3607,14 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-util", "tower", "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.4.2", "web-sys", ] @@ -3653,7 +3658,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.5.0", "web-sys", ] @@ -3935,6 +3940,7 @@ dependencies = [ "axum", "clap", "libc", + "reqwest 0.12.28", "rocm-core", "rocm-engine-protocol", "serde", @@ -3942,6 +3948,8 @@ dependencies = [ "sha2", "tokio", "ureq", + "utoipa", + "utoipa-swagger-ui", ] [[package]] @@ -5452,6 +5460,51 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "utoipa" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8765fe27aeff71012a3f90fa474cf1df863a7d0dd81ed25de4e63fff2544994f" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d935f1c83fdf8b88f09bbe050d0cea78ea4afee6d7b0317403228c1200b2c8ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1fd2f9615838885c6ff4115fde8a2b58825dc7b4617f78d29c167377a15c92" +dependencies = [ + "base64 0.23.1", + "mime_guess", + "regex", + "serde", + "serde_json", + "utoipa", + "utoipa-swagger-ui-vendored", + "zip", +] + +[[package]] +name = "utoipa-swagger-ui-vendored" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552a64d6a0d251ae28bc021838800943cce60c9ec2e60ff331f54e79f696268f" + [[package]] name = "uuid" version = "1.23.3" @@ -5596,6 +5649,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-streams" version = "0.5.0" diff --git a/MANIFEST.md b/MANIFEST.md index ca33e2290..5acc20a08 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -538,6 +538,10 @@ repository. | utf-8 | 0.7.6 | MIT OR Apache-2.0 | | utf8_iter | 1.0.4 | Apache-2.0 OR MIT | | utf8parse | 0.2.2 | Apache-2.0 OR MIT | +| utoipa | 6.0.0 | MIT OR Apache-2.0 | +| utoipa-gen | 6.0.1 | MIT OR Apache-2.0 | +| utoipa-swagger-ui | 10.0.2 | MIT OR Apache-2.0 | +| utoipa-swagger-ui-vendored | 0.2.0 | MIT OR Apache-2.0 | | uuid | 1.23.3 | Apache-2.0 OR MIT | | valuable | 0.1.1 | MIT | | version_check | 0.9.5 | MIT/Apache-2.0 | @@ -553,6 +557,7 @@ repository. | wasm-bindgen-macro | 0.2.125 | MIT OR Apache-2.0 | | wasm-bindgen-macro-support | 0.2.125 | MIT OR Apache-2.0 | | wasm-bindgen-shared | 0.2.125 | MIT OR Apache-2.0 | +| wasm-streams | 0.4.2 | MIT OR Apache-2.0 | | wasm-streams | 0.5.0 | MIT OR Apache-2.0 | | web-sys | 0.3.102 | MIT OR Apache-2.0 | | web-time | 1.1.0 | MIT OR Apache-2.0 | @@ -775,6 +780,14 @@ With the prek hooks installed this runs automatically whenever you commit a dependency-graph change, so the notices rarely go stale by hand; the hook skips if cargo-about is absent or not the pinned version. +Assets compiled in that are not crates — today, the Swagger UI bundle behind +`rocm services docs` — are listed in a "Bundled web assets" section maintained +by hand at the end of `about.hbs`. `cargo-about` cannot see them, so their +licenses bypass the `about.toml` check: when a dependency bump changes such an +asset, update that section from the asset's own upstream notices. A test in +`rocmd` fails when the served Swagger UI version is not the one the notices +name. + CI verifies the committed file is current with `cargo xtask tpn --check`; a new dependency under a license not listed in `about.toml` fails the check until the license is reviewed and added (or the dependency dropped). CI pins the diff --git a/README.md b/README.md index 09dca9adf..6633f470a 100644 --- a/README.md +++ b/README.md @@ -1251,8 +1251,9 @@ the running service, or stop it first to serve on the new port. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table -with the status, the full inference endpoint, the API-qualified model name, and a -quick smoke test (time to first token and approximate tokens per second). Control +with the status, the engine, the API-qualified model name, the full inference +endpoint, the command that serves the server's API docs (see API docs below), +and a quick smoke test (time to first token and approximate tokens per second). Control returns to your shell with the server still running; manage it later with `rocm services` (below). @@ -1477,6 +1478,7 @@ Manage background servers started with `--managed`: ``` rocm services list [--all] [--json] +rocm services docs [] [--port ] rocm services logs [--follow] rocm services stop --yes rocm services restart --yes @@ -1549,6 +1551,46 @@ started while a `prune` is scanning waits for the prune. `--json` prints the service records verbatim, for scripting and for the remote orchestration below. +#### API docs + +`rocm services docs` serves interactive API docs for the running model servers, +in the foreground, until you press Ctrl+C. `rocm serve` names the command next +to the endpoint, and `rocm services list` repeats it for each running server +(`--json` prints the records alone, without it): + +``` + endpoint http://127.0.0.1:11435/v1/chat/completions + docs rocm services docs +``` + +Run it to get the links: + +``` +$ rocm services docs +API docs for running model servers: http://127.0.0.1:11430/ + (): http://127.0.0.1:11430/services//docs/ +Press Ctrl+C to stop serving the docs; the model servers keep running. +``` + +Without a service id it links every running server. It refuses, and serves +nothing, when the named server is not running or when no server is. + +The page lists the routes the server answers (the model list, chat and text +completions, and the engine's health check), with request examples already +filled in with the served model. "Try it out" sends the request through the +docs page to the server, so it works for every engine without enabling +cross-origin access on the server; for a server started with an endpoint key, +enter the key under "Authorize" (it is kept only until the page is closed). +The root page lists every running server, judged as `rocm services list` judges +it; a server that stops or exits drops off about a second later. vLLM also documents its full API at +its own `/docs` page: `rocm serve` and `rocm services list` print it as +`engine docs`, and the docs page links to it. + +The docs listen on 127.0.0.1 only, answer browser requests only from their own +pages, cannot be framed by other sites, and work offline. They use port 11430, or a free port when 11430 is +taken; `--port ` picks one and fails if it is taken. Nothing keeps +running after Ctrl+C, and the model servers are never touched. + ### Remote machines (preview) Run a model on a different GPU machine and reach it from your own. Both machines diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index 175271e8c..d2528a21a 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -1934,6 +1934,217 @@ The following component(s) are licensed under Apache License 2.0: Apache License 2.0 ================================================================================ +The following component(s) are licensed under Apache License 2.0: + - utoipa-gen 6.0.1 + - utoipa-swagger-ui 10.0.2 + - utoipa 6.0.0 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================================ +Apache License 2.0 +================================================================================ + The following component(s) are licensed under Apache License 2.0: - rustls-platform-verifier 0.7.0 - unicode-linebreak 0.1.5 @@ -8421,6 +8632,7 @@ The following component(s) are licensed under Apache License 2.0: - unicode-ident 1.0.24 - utf8parse 0.2.2 - vte 0.15.0 + - wasm-streams 0.4.2 - wasm-streams 0.5.0 - winapi-i686-pc-windows-gnu 0.4.0 - winapi-x86_64-pc-windows-gnu 0.4.0 @@ -12124,3 +12336,130 @@ the following restrictions: 3. This notice may not be removed or altered from any source distribution. +================================================================================ +Bundled web assets +================================================================================ + +These components are not Rust crates; they are compiled into rocm-cli by a +crate listed above. Each license's full text appears above. + + - Swagger UI 5.32.6 + Licensed under Apache License 2.0. Compiled in by utoipa-swagger-ui + (through utoipa-swagger-ui-vendored) for the API docs of running model + servers. Its NOTICE file reads: + + swagger-ui + Copyright 2020-2021 SmartBear Software Inc. + + Swagger UI's bundle also contains third-party JavaScript. The notices it + retains, as distributed with Swagger UI 5.32.6 + (swagger-ui-bundle.js.LICENSE.txt in the swagger-ui-dist package; the + standalone preset's notices are a subset of these), read: + + /*! + Copyright (c) 2018 Jed Watson. + Licensed under the MIT License (MIT), see + http://jedwatson.github.io/classnames + */ + + /*! + * @description Recursive object extending + * @author Viacheslav Lotsmanov + * @license MIT + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Viacheslav Lotsmanov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + + /*! + * The buffer module from node.js, for the browser. + * + * @author Feross Aboukhadijeh + * @license MIT + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2021 Joachim Wester + * MIT license + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2022 Joachim Wester + * MIT licensed + */ + + /*! + * repeat-string + * + * Copyright (c) 2014-2015, Jon Schlinkert. + * Licensed under the MIT License. + */ + + /*! @license DOMPurify 3.4.0 | (c) Cure53 and other contributors | Released under the Apache license 2.0 and Mozilla Public License 2.0 | github.com/cure53/DOMPurify/blob/3.4.0/LICENSE */ + + /*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh */ + + /*! js-yaml 4.1.1 https://github.com/nodeca/js-yaml @license MIT */ + + /*! safe-buffer. MIT License. Feross Aboukhadijeh */ + + /** + * @license React + * react-dom.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * react.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * scheduler.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * use-sync-external-store-with-selector.production.js + * + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + diff --git a/about.hbs b/about.hbs index 078153459..bf281fc8a 100644 --- a/about.hbs +++ b/about.hbs @@ -30,3 +30,130 @@ The following component(s) are licensed under {{{name}}}: {{{text}}} {{/each}} + +================================================================================ +Bundled web assets +================================================================================ + +These components are not Rust crates; they are compiled into rocm-cli by a +crate listed above. Each license's full text appears above. + + - Swagger UI 5.32.6 + Licensed under Apache License 2.0. Compiled in by utoipa-swagger-ui + (through utoipa-swagger-ui-vendored) for the API docs of running model + servers. Its NOTICE file reads: + + swagger-ui + Copyright 2020-2021 SmartBear Software Inc. + + Swagger UI's bundle also contains third-party JavaScript. The notices it + retains, as distributed with Swagger UI 5.32.6 + (swagger-ui-bundle.js.LICENSE.txt in the swagger-ui-dist package; the + standalone preset's notices are a subset of these), read: + + /*! + Copyright (c) 2018 Jed Watson. + Licensed under the MIT License (MIT), see + http://jedwatson.github.io/classnames + */ + + /*! + * @description Recursive object extending + * @author Viacheslav Lotsmanov + * @license MIT + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Viacheslav Lotsmanov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + + /*! + * The buffer module from node.js, for the browser. + * + * @author Feross Aboukhadijeh + * @license MIT + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2021 Joachim Wester + * MIT license + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2022 Joachim Wester + * MIT licensed + */ + + /*! + * repeat-string + * + * Copyright (c) 2014-2015, Jon Schlinkert. + * Licensed under the MIT License. + */ + + /*! @license DOMPurify 3.4.0 | (c) Cure53 and other contributors | Released under the Apache license 2.0 and Mozilla Public License 2.0 | github.com/cure53/DOMPurify/blob/3.4.0/LICENSE */ + + /*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh */ + + /*! js-yaml 4.1.1 https://github.com/nodeca/js-yaml @license MIT */ + + /*! safe-buffer. MIT License. Feross Aboukhadijeh */ + + /** + * @license React + * react-dom.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * react.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * scheduler.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * use-sync-external-store-with-selector.production.js + * + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 0096036fe..9a0fec4ef 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -460,7 +460,8 @@ rocm model --verbose" /// Anthropic `/v1/messages` endpoint is published beside the OpenAI /// one (see `rocm services list`). By default the /// server runs as a managed background service and prints a deployment summary - /// (status, endpoint, model, and smoke-test metrics); use --verbose to stream engine + /// (status, endpoint, the `rocm services docs` command for its API docs, model, and + /// smoke-test metrics); use --verbose to stream engine /// logs in this terminal instead. When streaming, press Ctrl-D to detach and leave the /// server running, or Ctrl-C to stop it. If that stop answers that it could not confirm /// the server exited, it says so on stderr and exits with an error; if the stop itself @@ -1177,6 +1178,10 @@ pub(crate) enum ComfyuiCommand { pub(crate) enum ServicesCommand { /// Show currently running local model servers, and count the records that /// are no longer running. + /// + /// Each running server gets a `docs:` line naming the command that serves + /// its API docs (`rocm services docs `); a vLLM server also gets an + /// `engine docs:` line, the docs vLLM serves itself. List { /// Include failed, stopped, and old service records. #[arg(short, long)] @@ -1189,6 +1194,23 @@ pub(crate) enum ServicesCommand { #[arg(long)] json: bool, }, + /// Serve browsable API docs for running local model servers until Ctrl+C. + /// + /// Serves a loopback-only page (Swagger UI) per running server listing the + /// routes it answers, with a working "Try it out" that sends requests + /// through the page to the server. Prints the page links, then serves them + /// until Ctrl+C; nothing keeps running afterwards and the model servers + /// are not touched. Refuses when the named server is not running, or when + /// no server is. + Docs { + /// Service id from `rocm services list`; without it, every running + /// server's docs are linked. + service_id: Option, + /// Port to serve the docs on (127.0.0.1 only). Without it, 11430 is + /// used, or a free port when 11430 is taken. + #[arg(long)] + port: Option, + }, /// Show logs for a local model server. Logs { /// Service id from `rocm services list`. @@ -1437,6 +1459,25 @@ mod tests { } } + /// The API docs and each engine's `supported_routes` take an engine's + /// health route from one table; an engine `rocm serve` accepts but the + /// table lacks would get docs without a health route. + #[test] + fn every_supported_engine_has_a_health_route_and_no_other_engine_does() { + for engine in SUPPORTED_ENGINES { + assert!( + rocm_engine_protocol::engine_health_route(engine).is_some(), + "{engine} is missing from ENGINE_HEALTH_ROUTES" + ); + } + for (engine, _) in rocm_engine_protocol::ENGINE_HEALTH_ROUTES { + assert!( + SUPPORTED_ENGINES.contains(&engine), + "ENGINE_HEALTH_ROUTES names {engine}, which `rocm serve` does not accept" + ); + } + } + /// EAI-7362: the top-level `rocm --help` command list must be alphabetical, /// including clap's implicit `help` entry (it must sort into place, not be /// pinned last). diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index f3fc7f3fa..4c59bf1a9 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -68,6 +68,7 @@ mod service_hints; mod service_launch; mod service_records; mod services_cmd; +mod services_docs; mod services_text; mod setup_cmd; mod shell_quote; diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index d35d73869..5306baf00 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -1925,6 +1925,7 @@ mod tests { metrics, api_key: None, notes: Vec::new(), + docs_rows: Vec::new(), }); assert!(rendered.contains("time to first token n/a")); assert!(rendered.contains("throughput (approx) n/a")); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 2336899be..c6d01925b 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -1316,6 +1316,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { metrics, api_key: launched_key, notes, + docs_rows: crate::services_docs::launch_docs_rows(&paths, &report.service_id), }; // Same intentional one-time key display as `print_managed_launch_plain` // in service_launch/start.rs; see its rationale for why the tag below is @@ -1324,7 +1325,11 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { print!("{}", serve_summary::render_summary(&summary)); } else { spinner.clear(); - print_managed_launch_plain(&report, launched_key.as_deref()); + print_managed_launch_plain( + &report, + launched_key.as_deref(), + &crate::services_docs::launch_docs_rows(&paths, &report.service_id), + ); } return Ok(()); } diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index cd0ab34bc..a1a8fbc9d 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -68,6 +68,10 @@ pub(crate) struct DeploymentSummary { pub api_key: Option, /// GPU/device warnings folded in from the serve plan. pub notes: Vec, + /// The `docs` row naming `rocm services docs`, and an `engine docs` row + /// for an engine that serves docs itself, from + /// [`crate::services_docs::launch_docs_rows`]; shown under the endpoints. + pub docs_rows: Vec<(&'static str, String)>, } /// Render `Some(ttft)` as a human duration, or `n/a` when the probe did not @@ -138,6 +142,7 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { model_rows, vec![("endpoint", summary.chat_endpoint.clone())], anthropic_rows, + summary.docs_rows.clone(), vec![ ("time to first token", format_ttft(summary.metrics.ttft)), ("throughput (approx)", format_tps(summary.metrics.gen_tps)), @@ -468,9 +473,45 @@ mod tests { }, api_key: None, notes: Vec::new(), + docs_rows: Vec::new(), } } + // Pins where the rows go. Which rows a launched server gets is + // `services_docs::launch_docs_rows`; that the `docs` command serves the + // docs is scenario service-docs-01, and that the line `rocm serve` prints + // does is the GPU serve canaries' last step (serve-05, serve-07). + #[test] + fn summary_lists_the_docs_rows_right_after_the_endpoint() { + let summary = DeploymentSummary { + docs_rows: vec![ + ("docs", "rocm services docs vllm-qwen-1720000000".to_owned()), + ("engine docs", "http://127.0.0.1:1337/docs".to_owned()), + ], + ..base_summary() + }; + let rendered = render_summary(&summary); + let lines: Vec<&str> = rendered.lines().map(str::trim).collect(); + let endpoint_row = lines + .iter() + .position(|line| line.starts_with("endpoint ")) + .expect("endpoint row"); + assert_eq!( + &lines[endpoint_row + 1..endpoint_row + 3], + [ + "docs rocm services docs vllm-qwen-1720000000", + "engine docs http://127.0.0.1:1337/docs", + ], + "{rendered}" + ); + assert!( + !render_summary(&base_summary()) + .lines() + .any(|line| line.trim_start().starts_with("docs ")), + "no docs rows, no docs line" + ); + } + #[test] fn format_ttft_uses_ms_under_a_second_and_seconds_above() { assert_eq!(format_ttft(Some(Duration::from_millis(180))), "180 ms"); diff --git a/apps/rocm/src/service_launch/attach.rs b/apps/rocm/src/service_launch/attach.rs index d76a90e8d..4003f3919 100644 --- a/apps/rocm/src/service_launch/attach.rs +++ b/apps/rocm/src/service_launch/attach.rs @@ -104,6 +104,10 @@ pub(crate) fn run_attached_service( println!("model already being served"); println!(" service_id: {}", report.service_id); println!(" endpoint: {}", report.endpoint_url); + for (label, value) in crate::services_docs::launch_docs_rows(paths, &report.service_id) + { + println!(" {label}: {value}"); + } println!(" status: {}", report.status); println!(" logs: rocm logs {}", report.service_id); println!( @@ -138,8 +142,12 @@ pub(crate) fn run_attached_service( // printed as the "serve plan" block in `serve()`; extend it with the launch // coordinates and the streaming hint rather than repeating it. let endpoint = format!("{}/v1", format_http_base_url(host, port)); + let docs_rows = crate::services_docs::launch_docs_rows(paths, &service_id); println!(" service_id: {service_id}"); println!(" endpoint: {endpoint}"); + for (label, value) in &docs_rows { + println!(" {label}: {value}"); + } if let Some(key) = endpoint_api_key { // Same intentional one-time key display as `print_managed_launch_plain` // in `service_launch/start.rs`; see its rationale for why the tag below is @@ -168,6 +176,9 @@ pub(crate) fn run_attached_service( println!("detached - server still running"); println!(" service_id: {service_id}"); println!(" endpoint: {endpoint}"); + for (label, value) in &docs_rows { + println!(" {label}: {value}"); + } println!(" list: rocm services"); println!(" logs: rocm logs {service_id}"); println!(" stop: {}", service_hints::service_stop_hint(&service_id)); diff --git a/apps/rocm/src/service_launch/start.rs b/apps/rocm/src/service_launch/start.rs index 06cb93697..ff9d27d18 100644 --- a/apps/rocm/src/service_launch/start.rs +++ b/apps/rocm/src/service_launch/start.rs @@ -177,11 +177,18 @@ pub(crate) fn start_managed_service( pub(crate) fn print_managed_launch_plain( report: &ManagedLaunchReport, endpoint_api_key: Option<&str>, + docs_rows: &[(&str, String)], ) { + let print_docs_rows = || { + for (label, value) in docs_rows { + println!(" {label}: {value}"); + } + }; if report.already_running { println!("managed service already running"); println!(" service_id: {}", report.service_id); println!(" endpoint: {}", report.endpoint_url); + print_docs_rows(); println!(" status: {}", report.status); println!(" note: existing service detected; no second process spawned"); return; @@ -192,6 +199,7 @@ pub(crate) fn print_managed_launch_plain( println!(" process_pid: {child_pid}"); } println!(" endpoint: {}", report.endpoint_url); + print_docs_rows(); if let Some(key) = endpoint_api_key { // Intentional one-time display of a freshly generated API key to the // terminal so the user can copy it — the designed delivery channel diff --git a/apps/rocm/src/services_cmd.rs b/apps/rocm/src/services_cmd.rs index f2cbc5262..4cbcb68be 100644 --- a/apps/rocm/src/services_cmd.rs +++ b/apps/rocm/src/services_cmd.rs @@ -3,7 +3,8 @@ // SPDX-License-Identifier: MIT //! `rocm services`: the command handler (`services`), which dispatches the -//! `list`, `logs`, `stop`, `restart`, `remove` and `prune` subcommands, and +//! `list`, `docs` (to `services_docs.rs`), `logs`, `stop`, `restart`, +//! `remove` and `prune` subcommands, and //! `run_approved_service_action`, the approval-gated stop/restart path, with //! `finish_service_action`, which renders its result and fails a stop that //! could not confirm the server gone. @@ -60,6 +61,9 @@ pub(crate) fn services(command: Option) -> Result<()> { } Ok(()) } + ServicesCommand::Docs { service_id, port } => { + crate::services_docs::run(&paths, service_id.as_deref(), port) + } ServicesCommand::Logs { service_id, follow } => { if follow { follow_service_logs(&paths, &service_id) diff --git a/apps/rocm/src/services_docs.rs b/apps/rocm/src/services_docs.rs new file mode 100644 index 000000000..dbb6328ae --- /dev/null +++ b/apps/rocm/src/services_docs.rs @@ -0,0 +1,459 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! `rocm services docs []`: browsable API docs for the running model +//! servers, served on loopback by this process until Ctrl+C. +//! +//! The docs server itself is `rocmd`'s [`rocmd::start_api_docs_server`], run +//! in-process. It runs only while this command does: nothing is left behind +//! when it stops, and the model servers are not touched. It is given the +//! loader `rocm services list` uses, so the docs list the same servers as +//! running that the list does, and a server that stops or crashes while the +//! docs are open drops off them. +//! +//! [`docs_hint`] is the line `rocm serve` and `rocm services list` print for a +//! running server, naming this command; the `engine docs` line beside it is +//! [`ManagedServiceRecord::engine_docs_url`], the docs an engine serves itself +//! (vLLM's `/docs`). + +use std::io::Write as _; + +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, ManagedServiceRecord}; + +use crate::managed_service_status::load_managed_services; +use crate::services_text::validate_service_id; + +/// The command that serves `service_id`'s API docs, as printed after a +/// running server's endpoint. The same string is what scenario +/// `service-docs-01` runs, so the hint and the command cannot drift apart. +pub(crate) fn docs_hint(service_id: &str) -> String { + format!( + "rocm services docs {}", + crate::shell_quote::shell_quote(service_id) + ) +} + +/// The rows `rocm serve` prints under a server's endpoint: `docs`, naming +/// this command, and `engine docs` when the engine serves docs of its own. +/// The record is read for its engine and address; one that cannot be read +/// still gets the `docs` row, since the command reports that itself. +pub(crate) fn launch_docs_rows(paths: &AppPaths, service_id: &str) -> Vec<(&'static str, String)> { + let mut rows = vec![("docs", docs_hint(service_id))]; + if let Some(engine_docs) = ManagedServiceRecord::read_manifest(paths, service_id) + .ok() + .as_ref() + .and_then(ManagedServiceRecord::engine_docs_url) + { + rows.push(("engine docs", engine_docs)); + } + rows +} + +/// `rocm services docs []`. +pub(crate) fn run(paths: &AppPaths, service_id: Option<&str>, port: Option) -> Result<()> { + if let Some(service_id) = service_id { + validate_service_id(service_id)?; + } + let records = load_managed_services(paths)?; + let targets = docs_targets(&records, service_id)?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("failed to build the runtime that waits for Ctrl+C")?; + // Listen for Ctrl+C before printing the link: the handler is installed + // when the future is first polled, and a Ctrl+C before that would kill + // the command without its closing line. One poll here installs it. + let mut interrupted = std::pin::pin!(tokio::signal::ctrl_c()); + let early = runtime.block_on(async { + tokio::select! { + biased; + result = &mut interrupted => Some(result), + () = std::future::ready(()) => None, + } + }); + // Defensive: Ctrl+C has to land within this first poll, which no test + // can aim at. A Ctrl+C after it, during startup, is caught by the handler + // and ends the wait below once the links are printed. + if let Some(result) = early { + // Nothing has been served or printed, so there is nothing to report. + result.context("failed to wait for Ctrl+C")?; + return Ok(()); + } + let (server, fallback_note) = + start_docs(paths, port, rocmd::DEFAULT_API_DOCS_PORT, |paths, port| { + rocmd::start_api_docs_server(paths, port, services_loader()) + })?; + + // On stderr, so stdout carries only the command's own output (the links + // and how to stop), which a caller reads from a pipe. + if let Some(note) = fallback_note { + eprintln!("{note}"); + } + let mut out = std::io::stdout().lock(); + writeln!( + out, + "API docs for running model servers: {}/", + server.base_url + )?; + for record in &targets { + writeln!( + out, + " {} ({}): {}", + record.service_id, + record.served_model_name(), + server.service_docs_url(&record.service_id) + )?; + } + writeln!( + out, + "Press Ctrl+C to stop serving the docs; the model servers keep running." + )?; + // A caller reading the link from a pipe must see it now, not at exit. + out.flush().context("failed to write the API docs link")?; + drop(out); + + let waited = runtime + .block_on(&mut interrupted) + .context("failed to wait for Ctrl+C"); + server.stop(); + waited?; + println!("API docs stopped."); + Ok(()) +} + +/// The loader the docs server reads the running servers with: the one +/// `rocm services list` uses, which checks each server's process and +/// endpoint. +fn services_loader() -> rocmd::ServiceLoader { + std::sync::Arc::new(|paths: &AppPaths| load_managed_services(paths)) +} + +/// The running servers whose docs links are printed: the one named, or every +/// running server. Refuses when there is nothing to document, so the command +/// never serves an empty page silently. +fn docs_targets<'a>( + records: &'a [ManagedServiceRecord], + service_id: Option<&str>, +) -> Result> { + let live: Vec<&ManagedServiceRecord> = records.iter().filter(|r| r.is_live()).collect(); + match service_id { + Some(service_id) => { + if let Some(record) = live.iter().find(|r| r.service_id == service_id) { + return Ok(vec![record]); + } + match records.iter().find(|r| r.service_id == service_id) { + // Stopped servers are listed only with `--all`; the restart + // command is the one that list prints for this server. + Some(record) => bail!( + "Server `{service_id}` is not running (status: {}), so it has no API docs. \ + Start it again with `{}`; `rocm services list --all` shows it.", + record.status, + crate::service_hints::service_action_hint("restart", service_id) + ), + // The not-found error every service-id command shares. + None => Err(rocm_core::ManagedServiceNotFound { + service_id: service_id.to_owned(), + } + .into()), + } + } + None if live.is_empty() => bail!( + "No model server is running, so there are no API docs to serve. \ + Start one with `rocm serve `." + ), + None => Ok(live), + } +} + +/// Start the docs server on the requested port, or on `default_port` when +/// none was requested, falling back to a free port when that one is taken. +/// An explicitly requested port is never swapped for another. Returns the +/// server and, after a fallback, the line saying so. +fn start_docs( + paths: &AppPaths, + requested: Option, + default_port: u16, + start: impl Fn(AppPaths, u16) -> Result, +) -> Result<(S, Option)> { + if let Some(port) = requested { + return Ok((start(paths.clone(), port)?, None)); + } + match start(paths.clone(), default_port) { + Ok(server) => Ok((server, None)), + // Only a port that is in use is worked around; any other failure + // would recur on the free port and is reported as it is. + Err(error) if !is_addr_in_use(&error) => Err(error), + Err(error) => { + let server = start(paths.clone(), 0)?; + Ok(( + server, + Some(format!( + "note: port {default_port} is not available ({error:#}); serving on a free port" + )), + )) + } + } +} + +fn is_addr_in_use(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::AddrInUse) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + fn record(paths: &AppPaths, id: &str, status: &str) -> ManagedServiceRecord { + let mut record = ManagedServiceRecord::new( + paths, + id, + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 8000, + "managed", + 0, + None, + None, + None, + ); + record.status = status.to_owned(); + record + } + + #[test] + fn docs_targets_cover_every_state_a_server_can_be_in() { + let (root, paths) = crate::tests::test_paths("services-docs-targets"); + let records = vec![ + record(&paths, "svc-live", "ready"), + record(&paths, "svc-starting", "starting"), + record(&paths, "svc-stopped", "stopped"), + ]; + let ids = |targets: Vec<&ManagedServiceRecord>| -> Vec { + targets.iter().map(|r| r.service_id.clone()).collect() + }; + + assert_eq!( + ids(docs_targets(&records, None).unwrap()), + ["svc-live", "svc-starting"], + "every running server, and only those" + ); + assert_eq!( + ids(docs_targets(&records, Some("svc-live")).unwrap()), + ["svc-live"] + ); + let stopped = docs_targets(&records, Some("svc-stopped")).unwrap_err(); + assert_eq!( + stopped.to_string(), + "Server `svc-stopped` is not running (status: stopped), so it has no API docs. \ + Start it again with `rocm services restart svc-stopped --yes`; \ + `rocm services list --all` shows it." + ); + let unknown = docs_targets(&records, Some("nope")).unwrap_err(); + assert_eq!( + unknown.to_string(), + "No local server named `nope`. See `rocm services list --all`." + ); + let none_running = docs_targets(&records[2..], None).unwrap_err(); + assert!( + none_running + .to_string() + .starts_with("No model server is running"), + "{none_running}" + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn a_taken_default_port_falls_back_to_a_free_one_but_a_requested_port_does_not() { + let (root, paths) = crate::tests::test_paths("services-docs-ports"); + let tried = RefCell::new(Vec::new()); + let taken_default = |_: AppPaths, port: u16| -> Result { + tried.borrow_mut().push(port); + if port == 11_430 { + return Err(anyhow::Error::from(std::io::Error::from( + std::io::ErrorKind::AddrInUse, + )) + .context("failed to bind API docs server to 127.0.0.1:11430")); + } + Ok(port) + }; + + let (bound, note) = start_docs(&paths, None, 11_430, taken_default).unwrap(); + assert_eq!(bound, 0, "fell back to a free port"); + assert_eq!(*tried.borrow(), [11_430, 0]); + assert_eq!( + note.as_deref(), + Some( + "note: port 11430 is not available (failed to bind API docs server to \ + 127.0.0.1:11430: address in use); serving on a free port" + ) + ); + + tried.borrow_mut().clear(); + let error = start_docs(&paths, Some(11_430), 11_430, taken_default).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to bind API docs server to 127.0.0.1:11430" + ); + assert_eq!( + *tried.borrow(), + [11_430], + "a requested port is never swapped" + ); + + tried.borrow_mut().clear(); + let (bound, note) = start_docs(&paths, None, 11_431, taken_default).unwrap(); + assert_eq!( + (bound, note), + (11_431, None), + "a free default port is used as is" + ); + + // A failure that is not the port being taken would recur on any + // port, so it is reported rather than retried elsewhere. + tried.borrow_mut().clear(); + let broken = |_: AppPaths, port: u16| -> Result { + tried.borrow_mut().push(port); + bail!("failed to build the API docs runtime") + }; + let error = start_docs(&paths, None, 11_430, broken).unwrap_err(); + assert_eq!(error.to_string(), "failed to build the API docs runtime"); + assert_eq!(*tried.borrow(), [11_430], "no fallback for other errors"); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn a_launched_server_gets_engine_docs_only_when_its_engine_serves_them() { + let (root, paths) = crate::tests::test_paths("services-docs-launch-rows"); + record(&paths, "svc-vllm", "ready").write().unwrap(); + let mut lemonade = record(&paths, "svc-lemonade", "ready"); + lemonade.engine = "lemonade".to_owned(); + lemonade.write().unwrap(); + + assert_eq!( + launch_docs_rows(&paths, "svc-vllm"), + [ + ("docs", "rocm services docs svc-vllm".to_owned()), + ("engine docs", "http://127.0.0.1:8000/docs".to_owned()), + ] + ); + assert_eq!( + launch_docs_rows(&paths, "svc-lemonade"), + [("docs", "rocm services docs svc-lemonade".to_owned())] + ); + assert_eq!( + launch_docs_rows(&paths, "svc-missing"), + [("docs", "rocm services docs svc-missing".to_owned())], + "an unreadable record still names the command, which reports it" + ); + let _ = std::fs::remove_dir_all(root); + } + + /// `rocmd` cannot call the engines; `rocm` links them, so the check that + /// the docs describe the routes each engine reports lives here. Each + /// engine is asked for its `Endpoint` answer the way the CLI asks it, + /// against a state file like the one its launch writes. + #[test] + fn documented_routes_are_the_routes_each_engine_reports() { + let (root, paths) = crate::tests::test_paths("services-docs-engine-routes"); + paths.ensure().unwrap(); + let service_id = "svc-routes"; + let mut reported = Vec::new(); + for engine in crate::cli::SUPPORTED_ENGINES { + let state_dir = paths.engine_state_dir(engine); + std::fs::create_dir_all(&state_dir).unwrap(); + std::fs::write( + state_dir.join(format!("{service_id}.json")), + r#"{"status":"running","host":"127.0.0.1","port":9,"endpoint_url":"http://127.0.0.1:9/v1"}"#, + ) + .unwrap(); + let endpoint: rocm_engine_protocol::EndpointResponse = + crate::engine_ipc::engine_request( + Some(&paths), + engine, + rocm_engine_protocol::EngineMethod::Endpoint, + &rocm_engine_protocol::EndpointRequest { + service_id: service_id.to_owned(), + }, + ) + .unwrap_or_else(|error| panic!("{engine}: {error:#}")); + reported.push((engine, endpoint.supported_routes)); + } + let _ = std::fs::remove_dir_all(root); + + for (engine, routes) in &reported { + assert_eq!(&rocmd::documented_route_paths(engine), routes, "{engine}"); + } + // Lemonade's health route differs from the others: an engine + // reporting another engine's routes cannot pass for its own. + assert_ne!(reported[0].1, reported[1].1); + } + + /// A server whose process died, though its record still says "ready", + /// is not running to the docs, as it is not to `rocm services list`. + #[test] + fn a_server_whose_process_died_has_no_docs() { + let (root, paths) = crate::tests::test_paths("services-docs-dead-pid"); + paths.ensure().unwrap(); + std::fs::create_dir_all(paths.services_dir()).unwrap(); + let mut exited = if cfg!(windows) { + std::process::Command::new("cmd") + .args(["/C", "exit 0"]) + .spawn() + .unwrap() + } else { + std::process::Command::new("true").spawn().unwrap() + }; + exited.wait().unwrap(); + // Nothing listens on the recorded port either. + let dead_port = std::net::TcpListener::bind(("127.0.0.1", 0)) + .unwrap() + .local_addr() + .unwrap() + .port(); + let mut crashed = record(&paths, "svc-crashed", "ready"); + crashed.port = dead_port; + crashed.supervisor_pid = exited.id(); + crashed.write().unwrap(); + assert!(crashed.is_live(), "premise: the record still says it runs"); + + let server = rocmd::start_api_docs_server(paths, 0, services_loader()).unwrap(); + // Not proxied: the docs are on loopback. + let agent = ureq::AgentBuilder::new().try_proxy_from_env(false).build(); + let index = agent + .get(&format!("{}/", server.base_url)) + .call() + .unwrap() + .into_string() + .unwrap(); + let (status, page) = match agent.get(&server.service_docs_url("svc-crashed")).call() { + Ok(response) => (response.status(), response.into_string().unwrap()), + Err(ureq::Error::Status(status, response)) => (status, response.into_string().unwrap()), + Err(error) => panic!("docs request failed: {error}"), + }; + server.stop(); + let _ = std::fs::remove_dir_all(root); + + assert!(!index.contains("svc-crashed"), "{index}"); + assert!(index.contains("No model server is running"), "{index}"); + assert_eq!(status, 404); + assert!(page.contains("is not running"), "{page}"); + } + + #[test] + fn the_docs_hint_names_this_command_with_the_server_id() { + // The hint text only; that the named command serves the docs is + // scenario service-docs-01, which runs the line `rocm services list` + // printed. + assert_eq!(docs_hint("svc-1"), "rocm services docs svc-1"); + } +} diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index cd6af59fb..788b4a30a 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -463,15 +463,22 @@ pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result Result<()> { + let mut rendered = Vec::new(); + for (name, engine, status) in [ + ("services-docs-vllm", "vllm", "starting"), + ("services-docs-lemonade", "lemonade", "starting"), + ("services-docs-stopped", "vllm", "stopped"), + ] { + let (root, paths) = test_paths(name); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-x", + engine, + "qwen", + "Qwen/Qwen3.5", + "0.0.0.0", + 11436, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = status.to_owned(); + record.write()?; + rendered.push(render_services_text(&paths, true)?); + let _ = fs::remove_dir_all(root); + } + + let [vllm, lemonade, stopped] = &rendered[..] else { + unreachable!() + }; + assert!( + vllm.contains("\n docs: rocm services docs svc-x\n"), + "{vllm}" + ); + assert!( + vllm.contains("\n engine docs: http://127.0.0.1:11436/docs\n"), + "{vllm}" + ); + assert!( + lemonade.contains("\n docs: rocm services docs svc-x\n"), + "{lemonade}" + ); + assert!(!lemonade.contains("engine docs:"), "{lemonade}"); + assert!(stopped.contains("status: stopped"), "{stopped}"); + assert!(!stopped.contains("docs:"), "{stopped}"); + Ok(()) + } + #[test] fn log_tail_yields_only_new_complete_segments() -> Result<()> { let (root, paths) = test_paths("log-tail"); diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 70587d745..3821f4e04 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -17,6 +17,11 @@ e2e-test-hooks = ["rocm-core/e2e-test-hooks"] anyhow.workspace = true async-stream.workspace = true axum.workspace = true +# Forwards the API docs' try-it requests to the model server, over plain +# HTTP to the address its record names (loopback for a server bound to every +# interface), so no TLS stack; `stream` relays streamed completions as they +# arrive. +reqwest = { version = "0.12", default-features = false, features = ["stream"] } clap.workspace = true rocm-core = { path = "../../crates/rocm-core" } rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol" } @@ -25,6 +30,11 @@ serde_json.workspace = true sha2 = "0.10" tokio.workspace = true ureq = { version = "2.12", features = ["native-certs"] } +# API docs for running model servers: OpenAPI model + Swagger UI. `vendored` +# compiles the Swagger UI assets in, so the build downloads nothing and the +# page works offline. +utoipa = "6" +utoipa-swagger-ui = { version = "10", default-features = false, features = ["vendored"] } [dev-dependencies] # The fake engine child both engine IPC callers are tested against. diff --git a/apps/rocmd/src/anthropic_proxy.rs b/apps/rocmd/src/anthropic_proxy.rs index 76486a915..90f769846 100644 --- a/apps/rocmd/src/anthropic_proxy.rs +++ b/apps/rocmd/src/anthropic_proxy.rs @@ -82,12 +82,7 @@ impl AnthropicProxyConfig { /// Where to reach the engine from this machine, even when it listens on a /// wildcard address. fn upstream_url(&self) -> String { - let host = match self.host.as_str() { - "0.0.0.0" => "127.0.0.1", - "::" => "::1", - other => other, - }; - format_http_base_url(host, self.engine_port) + format_http_base_url(rocm_core::reachable_host(&self.host), self.engine_port) } } @@ -1656,6 +1651,10 @@ mod tests { }; assert_eq!(config("0.0.0.0").upstream_url(), "http://127.0.0.1:8000"); assert_eq!(config("::").upstream_url(), "http://[::1]:8000"); + assert_eq!( + config("0:0:0:0:0:0:0:0").upstream_url(), + "http://[::1]:8000" + ); assert_eq!(config("127.0.0.1").upstream_url(), "http://127.0.0.1:8000"); } diff --git a/apps/rocmd/src/api_docs.rs b/apps/rocmd/src/api_docs.rs new file mode 100644 index 000000000..1666cdd01 --- /dev/null +++ b/apps/rocmd/src/api_docs.rs @@ -0,0 +1,1955 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! API docs for every running model server, served on loopback for as long as +//! `rocm services docs` runs in the foreground. +//! +//! Of the servers rocm-cli runs, only vLLM generates its own docs; Lemonade and +//! the llama.cpp server behind it serve none. So this server describes each +//! running server from the routes rocm-cli knows it serves ([`routes_for`]), +//! renders that with Swagger UI, and forwards the page's "Try it out" requests +//! to the server through `/services//api/...`. Forwarding keeps try-it +//! same-origin, so it works on every engine whatever that engine's own +//! cross-origin settings are; Lemonade, for one, only admits browser requests +//! from its own origin. +//! +//! It lives here, beside the daemon, rather than in `rocm`, which runs it +//! in-process: `rocm` depends on `rocmd`, so the HTTP stack it needs stays in +//! one crate. It runs only while `rocm services docs` does; nothing in the +//! background serves it. +//! +//! Which servers are running comes from the [`ServiceLoader`] the caller +//! passes, asked again on each request (at most once per +//! [`SERVICE_LIST_TTL`]). `rocm` passes the loader `rocm services list` uses, +//! which checks each server's process and endpoint, so a server that stops or +//! crashes while the docs are open drops off them. + +use anyhow::{Context, Result, bail}; +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::{DefaultBodyLimit, Path, Request, State}; +use axum::http::{HeaderMap, HeaderValue, Method, StatusCode, Uri, header}; +use axum::middleware::{self, Next}; +use axum::response::{Html, IntoResponse, Response}; +use axum::routing::{any, get}; +use rocm_core::{AppPaths, DEFAULT_LOCAL_HOST, ManagedServiceRecord}; +use serde_json::{Value, json}; +use std::fmt::Write as _; +use std::net::IpAddr; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::net::TcpListener; +use utoipa::openapi::path::{HttpMethod, OperationBuilder, PathItem, PathsBuilder}; +use utoipa::openapi::request_body::RequestBodyBuilder; +use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityRequirement, SecurityScheme}; +use utoipa::openapi::{ + ComponentsBuilder, ContentBuilder, InfoBuilder, OpenApi, OpenApiBuilder, Required, + ResponseBuilder, ServerBuilder, +}; + +/// Largest request body the try-it forwarder accepts. Generous enough for a +/// long chat history or a large prompt; the docs server is loopback-only, so +/// this bounds memory, not abuse from elsewhere. +const FORWARD_BODY_LIMIT_BYTES: usize = 64 * 1024 * 1024; + +/// How long the forwarder waits for the model server to accept a connection. +/// There is deliberately no overall timeout: a long completion may stream for +/// minutes. +const FORWARD_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +/// The OpenAPI security scheme name used for servers started with an endpoint +/// key; Swagger UI's "Authorize" box fills it in. +const ENDPOINT_KEY_SCHEME: &str = "endpoint_key"; + +/// How long a stopping docs server lets in-flight requests finish. A try-it +/// answer may stream for minutes and the forwarder sets no overall timeout, so +/// without this bound an open stream would hold up Ctrl+C. +const SHUTDOWN_GRACE: Duration = Duration::from_secs(2); + +/// How long one read of the running servers is reused. Opening a docs page +/// sends several requests at once, and the loader `rocm` passes probes each +/// server's endpoint; requests that arrive while a read is under way wait for +/// it, so a page load costs one read. A server that stops drops off once the +/// read after it stopped completes: at most this long after the last read, +/// plus the time the next read takes. +const SERVICE_LIST_TTL: Duration = Duration::from_secs(1); + +/// `Content-Security-Policy` on every page this server serves itself: no +/// other site may frame it, so a framed docs page cannot be clicked through +/// into sending a try-it request from this origin. +const PAGE_CSP: &str = "frame-ancestors 'none'"; + +/// `Content-Security-Policy` on forwarded answers: as [`PAGE_CSP`], and +/// sandboxed, since the answer comes from the model server but is served from +/// the docs' origin. +const FORWARDED_CSP: &str = "sandbox; frame-ancestors 'none'"; + +/// Reads every service record, refreshed however the caller judges a server +/// to be running; the docs list the ones [`ManagedServiceRecord::is_live`] +/// accepts afterwards. +pub type ServiceLoader = Arc Result> + Send + Sync>; + +/// The port `rocm services docs` serves on unless told otherwise. Kept clear +/// of `DEFAULT_LOCAL_PORT` and the ports above it that consecutive +/// `rocm serve` launches pick. +pub const DEFAULT_API_DOCS_PORT: u16 = 11_430; + +/// A running docs server. It runs on a thread and runtime of its own, so the +/// caller's thread stays free to wait for Ctrl+C. +pub struct ApiDocsServer { + /// `http://127.0.0.1:`, the port actually bound. + pub base_url: String, + shutdown: tokio::sync::oneshot::Sender<()>, + thread: std::thread::JoinHandle<()>, +} + +impl ApiDocsServer { + /// The API docs page of one server. + #[must_use] + pub fn service_docs_url(&self, service_id: &str) -> String { + format!("{}{}/docs/", self.base_url, service_path(service_id)) + } + + /// Stop accepting requests and wait for the server thread to finish. + pub fn stop(self) { + let _ = self.shutdown.send(()); + let _ = self.thread.join(); + } +} + +/// The last read of the running servers, and when it was made. +type CachedServices = Option<(Instant, Arc>)>; + +#[derive(Clone)] +struct DocsState { + paths: Arc, + load: ServiceLoader, + /// Held across a read, so concurrent requests share one. + cached: Arc>, + service_list_ttl: Duration, + client: reqwest::Client, + /// The origins of this server's own pages, the only ones whose browser + /// requests are admitted. + own_origins: Arc<[String]>, +} + +/// Serve API docs for the running model servers on `127.0.0.1:`; port +/// 0 binds a free one, named in [`ApiDocsServer::base_url`]. `load` reads the +/// service records; see [`ServiceLoader`]. +/// +/// # Errors +/// Fails when the port cannot be bound or the server cannot start. +pub fn start_api_docs_server( + paths: AppPaths, + port: u16, + load: ServiceLoader, +) -> Result { + start_with(paths, port, load, FORWARD_CONNECT_TIMEOUT, SERVICE_LIST_TTL) +} + +fn start_with( + paths: AppPaths, + port: u16, + load: ServiceLoader, + connect_timeout: Duration, + service_list_ttl: Duration, +) -> Result { + let listener = std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, port)).with_context(|| { + format!("failed to bind API docs server to {DEFAULT_LOCAL_HOST}:{port}") + })?; + let addr = listener + .local_addr() + .context("failed to read API docs listener address")?; + if !addr.ip().is_loopback() { + bail!("API docs server must bind to loopback; resolved address was {addr}"); + } + listener + .set_nonblocking(true) + .context("failed to configure the API docs listener")?; + let bound = addr.port(); + let app = router(DocsState { + paths: Arc::new(paths), + load, + cached: Arc::new(tokio::sync::Mutex::new(None)), + service_list_ttl, + client: forwarding_client(connect_timeout)?, + own_origins: [ + format!("http://{DEFAULT_LOCAL_HOST}:{bound}"), + format!("http://localhost:{bound}"), + ] + .into(), + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("failed to build the API docs runtime")?; + // Registered here rather than on the server thread, so a failure is this + // call's error instead of a thread that ends while the caller waits. + let listener = { + let _entered = runtime.enter(); + TcpListener::from_std(listener).context("failed to register the API docs listener")? + }; + let (shutdown, shutdown_received) = tokio::sync::oneshot::channel::<()>(); + let thread = std::thread::Builder::new() + .name("rocmd-api-docs".to_owned()) + .spawn(move || { + runtime.block_on(async move { + // `axum::serve` retries failed accepts rather than returning + // an error, so this ends only when the caller stops it. + if let Err(error) = serve_until_stopped(listener, app, shutdown_received).await { + eprintln!("API docs server stopped: {error}"); + } + }); + }) + .context("failed to start the API docs thread")?; + Ok(ApiDocsServer { + base_url: format!("http://{addr}"), + shutdown, + thread, + }) +} + +/// Serve until `shutdown` fires, then give in-flight requests +/// [`SHUTDOWN_GRACE`] to finish before dropping them. +async fn serve_until_stopped( + listener: TcpListener, + app: Router, + shutdown: tokio::sync::oneshot::Receiver<()>, +) -> Result<()> { + let stopping = Arc::new(tokio::sync::Notify::new()); + let stop_signal = Arc::clone(&stopping); + let serve = axum::serve(listener, app).with_graceful_shutdown(async move { + let _ = shutdown.await; + stop_signal.notify_one(); + }); + tokio::select! { + served = serve => served.map_err(anyhow::Error::from), + () = async { + stopping.notified().await; + tokio::time::sleep(SHUTDOWN_GRACE).await; + } => Ok(()), + } +} + +/// The client the forwarder sends try-it requests with. +fn forwarding_client(connect_timeout: Duration) -> Result { + reqwest::Client::builder() + .connect_timeout(connect_timeout) + // The model server is on this machine. A proxy configured for the + // user's outbound traffic (`HTTP_PROXY`, `ALL_PROXY`) would carry the + // prompt and the endpoint key off it, or fail the request outright. + .no_proxy() + // A redirect is not followed: the browser gets the model server's own + // status. The routes documented here never redirect. + .redirect(reqwest::redirect::Policy::none()) + .build() + .context("failed to build the API docs forwarding client") +} + +fn router(state: DocsState) -> Router { + Router::new() + .route("/", get(index_page)) + .route("/services/{id}/openapi.json", get(service_openapi_json)) + .route("/services/{id}/docs", get(docs_redirect)) + .route("/services/{id}/docs/", get(docs_index)) + .route("/services/{id}/docs/{*file}", get(docs_file)) + .route( + "/services/{id}/api/{*path}", + any(forward).layer(DefaultBodyLimit::max(FORWARD_BODY_LIMIT_BYTES)), + ) + .layer(middleware::from_fn_with_state( + state.clone(), + require_loopback_host, + )) + .with_state(state) +} + +/// Refuse requests whose `Host` is not a loopback name, and browser requests +/// sent from any page but this server's own. The listener is already +/// loopback-only; the `Host` check closes DNS rebinding, where a web page +/// resolves its own hostname to 127.0.0.1. A browser request is recognised by +/// its `Origin`, which a page's `fetch` and form posts carry and which must +/// name this server, or, when it has none (a link, an image, a top-level +/// navigation), by `Sec-Fetch-Site`, which must say the request came from +/// this server's own pages or from the user typing the address. Requests from +/// other programs (curl) carry neither and are admitted. So no other page the +/// user has open, another local web app included, can send a request through +/// the forwarder. +/// +/// Every answer is marked so no other site can frame it or have it sniffed +/// into a different type. +async fn require_loopback_host( + State(state): State, + request: Request, + next: Next, +) -> Response { + if !request_is_admitted(request.headers(), &state.own_origins) { + return mark_as_not_frameable( + ( + StatusCode::FORBIDDEN, + "API docs answer only their own pages, on 127.0.0.1 or localhost\n", + ) + .into_response(), + ); + } + mark_as_not_frameable(next.run(request).await) +} + +fn request_is_admitted(headers: &HeaderMap, own_origins: &[String]) -> bool { + let text = |name| headers.get(name).map(|value: &HeaderValue| value.to_str()); + let host_allowed = + matches!(text(header::HOST), Some(Ok(host)) if host_header_is_loopback(host)); + let browser_allowed = match (text(header::ORIGIN), text(SEC_FETCH_SITE)) { + (Some(origin), _) => origin.is_ok_and(|origin| origin_is_own(origin, own_origins)), + (None, Some(site)) => site.is_ok_and(fetch_site_is_own), + (None, None) => true, + }; + host_allowed && browser_allowed +} + +/// The fetch-metadata header browsers send with every request, `Origin` or +/// not. +const SEC_FETCH_SITE: header::HeaderName = header::HeaderName::from_static("sec-fetch-site"); + +/// `same-origin` is a request from this server's own pages; `none` is one the +/// user started (typing the address, a bookmark). `same-site` is refused too: +/// another port on 127.0.0.1 is the same site. +fn fetch_site_is_own(site: &str) -> bool { + matches!(site, "same-origin" | "none") +} + +fn mark_as_not_frameable(mut response: Response) -> Response { + let headers = response.headers_mut(); + headers + .entry(header::CONTENT_SECURITY_POLICY) + .or_insert(HeaderValue::from_static(PAGE_CSP)); + headers.insert(header::X_FRAME_OPTIONS, HeaderValue::from_static("DENY")); + headers.insert( + header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ); + response +} + +fn host_header_is_loopback(host: &str) -> bool { + let name = if let Some(rest) = host.strip_prefix('[') { + rest.split(']').next().unwrap_or_default() + } else { + host.rsplit_once(':').map_or(host, |(name, _)| name) + }; + name.eq_ignore_ascii_case("localhost") + || name.parse::().is_ok_and(|ip| ip.is_loopback()) +} + +/// Whether an `Origin` header names one of this server's own pages. Browsers +/// send the scheme and host lowercased and the port only when it is not the +/// scheme's default, so an exact comparison is enough. +fn origin_is_own(origin: &str, own_origins: &[String]) -> bool { + own_origins.iter().any(|own| own == origin) +} + +/// Every running server, as the [`ServiceLoader`] judges it. A failure to +/// read the records is logged and returned, so a page can say the records +/// could not be read rather than claim that nothing is running. The loader +/// may probe each server, so it runs off the async thread. Requests that +/// arrive during a read wait for it rather than starting their own, and reuse +/// its result when it succeeds; a failed read is not remembered, so each +/// waiter then reads again in turn. The loader must therefore bound its own +/// probes, as the one `rocm` passes does: each server's liveness check and +/// inference probe have their own timeouts (750 ms and +/// `rocm_core::INFERENCE_PROBE_TIMEOUT`, 8 s), so a read takes at most about +/// nine seconds per server, servers probed in turn. A timeout here would only +/// abandon the read, not stop it. +async fn running_services(state: &DocsState) -> Result>> { + let mut cached = state.cached.lock().await; + if let Some((read_at, services)) = cached.as_ref() + && read_at.elapsed() < state.service_list_ttl + { + return Ok(Arc::clone(services)); + } + let load = Arc::clone(&state.load); + let paths = Arc::clone(&state.paths); + let loaded = tokio::task::spawn_blocking(move || load(&paths)) + .await + .context("the service records read stopped") + .and_then(|loaded| loaded) + .inspect_err(|error| { + eprintln!("API docs could not read the service records: {error:#}"); + })?; + let services: Arc> = Arc::new( + loaded + .into_iter() + .filter(ManagedServiceRecord::is_live) + .collect(), + ); + *cached = Some((Instant::now(), Arc::clone(&services))); + Ok(services) +} + +/// The running server with this id. Looked up by matching the id against the +/// records rather than by building a path from it, so a crafted id cannot +/// reach outside the services folder. +async fn find_running_service( + state: &DocsState, + service_id: &str, +) -> Result> { + Ok(running_services(state) + .await? + .iter() + .find(|record| record.service_id == service_id) + .cloned()) +} + +/// `/services/`, with the id percent-encoded: an id may hold characters +/// (a space, `#`, `?`, `%`) that would otherwise end or change the path. +fn service_path(service_id: &str) -> String { + let mut path = String::from("/services/"); + for byte in service_id.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + path.push(char::from(byte)); + } else { + let _ = write!(path, "%{byte:02X}"); + } + } + path +} + +/// The answer for a server lookup that could not read the records: 503, since +/// the server may well be running. +fn records_unreadable(error: &anyhow::Error) -> Response { + ( + StatusCode::SERVICE_UNAVAILABLE, + axum::Json(json!({ + "error": format!("could not read the service records: {error:#}"), + })), + ) + .into_response() +} + +fn not_running_page(service_id: &str) -> Response { + let id = html_escape(service_id); + ( + StatusCode::NOT_FOUND, + Html(format!( + "Server not running\ +

Server {id} is not running

\ +

Its API docs are available while it runs. \ + Running servers

" + )), + ) + .into_response() +} + +async fn index_page(State(state): State) -> Response { + let services = match running_services(&state).await { + Ok(services) => services, + Err(error) => return records_unreadable(&error), + }; + let mut rows = String::new(); + for record in services.iter() { + let _ = write!( + rows, + "
  • {model} ({engine}, {id}, {endpoint})
  • ", + // Percent-encoded, so it holds no character HTML treats specially. + href = service_path(&record.service_id), + id = html_escape(&record.service_id), + model = html_escape(record.served_model_name()), + engine = html_escape(&record.engine), + endpoint = html_escape(&record.endpoint_url), + ); + } + let body = if services.is_empty() { + "

    No model server is running. Start one with rocm serve.

    ".to_owned() + } else { + format!("
      {rows}
    ") + }; + Html(format!( + "ROCm CLI API docs\ +

    Running model servers

    {body}" + )) + .into_response() +} + +async fn service_openapi_json( + State(state): State, + Path(service_id): Path, +) -> Response { + match find_running_service(&state, &service_id).await { + Ok(Some(record)) => axum::Json(service_openapi(&record)).into_response(), + Err(error) => records_unreadable(&error), + Ok(None) => ( + StatusCode::NOT_FOUND, + axum::Json(json!({ + "error": format!("server {service_id} is not running"), + })), + ) + .into_response(), + } +} + +/// `/services//docs` to the page itself, whose assets load relative to +/// the trailing slash. Only a running server's id is redirected, and +/// percent-encoded again, since it comes decoded from the request. The +/// redirect is temporary, since the server it names can stop. +async fn docs_redirect(State(state): State, Path(service_id): Path) -> Response { + match find_running_service(&state, &service_id).await { + Ok(Some(record)) => { + match HeaderValue::try_from(format!("{}/docs/", service_path(&record.service_id))) { + Ok(location) => ( + StatusCode::TEMPORARY_REDIRECT, + [(header::LOCATION, location)], + ) + .into_response(), + Err(_) => not_running_page(&service_id), + } + } + Ok(None) => not_running_page(&service_id), + Err(error) => records_unreadable(&error), + } +} + +async fn docs_index(State(state): State, Path(service_id): Path) -> Response { + serve_swagger_file(&state, &service_id, "").await +} + +async fn docs_file( + State(state): State, + Path((service_id, file)): Path<(String, String)>, +) -> Response { + serve_swagger_file(&state, &service_id, &file).await +} + +async fn serve_swagger_file(state: &DocsState, service_id: &str, file: &str) -> Response { + match find_running_service(state, service_id).await { + Ok(Some(_)) => {} + Ok(None) => return not_running_page(service_id), + Err(error) => return records_unreadable(&error), + } + let config = Arc::new( + utoipa_swagger_ui::Config::new([format!("{}/openapi.json", service_path(service_id))]) + .try_it_out_enabled(true) + // The endpoint key typed into "Authorize" lasts until the page is + // closed; it is not saved in the browser, where it would outlive + // the server it belongs to. + .persist_authorization(false), + ); + match utoipa_swagger_ui::serve(file, config) { + Ok(Some(swagger_file)) => ( + [(header::CONTENT_TYPE, swagger_file.content_type)], + swagger_file.bytes.into_owned(), + ) + .into_response(), + Ok(None) => StatusCode::NOT_FOUND.into_response(), + Err(error) => (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(), + } +} + +/// Relay one try-it request to the model server and stream its answer back. +async fn forward( + State(state): State, + Path((service_id, _decoded_path)): Path<(String, String)>, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let record = match find_running_service(&state, &service_id).await { + Ok(Some(record)) => record, + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + axum::Json(json!({ "error": format!("server {service_id} is not running") })), + ) + .into_response(); + } + Err(error) => return records_unreadable(&error), + }; + // The path exactly as the browser sent it. The extracted one is + // percent-decoded: an encoded `?` or `/` in it would turn into a query or + // an extra segment, and the request would reach a different route. + let Some(path) = raw_forwarded_path(uri.path()) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let mut target = format!("{}/{path}", record.local_base_url()); + if let Some(query) = uri.query() { + target.push('?'); + target.push_str(query); + } + let mut request = state.client.request(method, &target).body(body); + // Only what the routes need: the body's type, the answer the caller + // accepts, and the endpoint key the user typed into "Authorize". Nothing + // the browser adds on its own (cookies, origin) reaches the model server. + for name in [header::CONTENT_TYPE, header::ACCEPT, header::AUTHORIZATION] { + if let Some(value) = headers.get(&name) { + request = request.header(name, value); + } + } + match request.send().await { + Ok(answer) => { + let status = answer.status(); + let content_type = answer.headers().get(header::CONTENT_TYPE).cloned(); + let mut response = Response::new(Body::from_stream(answer.bytes_stream())); + *response.status_mut() = status; + let headers = response.headers_mut(); + if let Some(content_type) = content_type { + headers.insert(header::CONTENT_TYPE, content_type); + } + // The answer comes from the model server but is served from the + // docs' origin. A page in it (llama.cpp's own UI, say) must not + // run there: it is shown as data, never as a document of this + // origin. `nosniff` is added for every answer, by + // `require_loopback_host`. + headers.insert( + header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static(FORWARDED_CSP), + ); + response + } + Err(error) => ( + StatusCode::BAD_GATEWAY, + [( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + )], + json!({ + "error": format!( + "could not reach server {service_id} at {}: {error}", + record.local_base_url() + ), + }) + .to_string(), + ) + .into_response(), + } +} + +/// The part of `/services//api/` after `api/`, still percent-encoded. +fn raw_forwarded_path(request_path: &str) -> Option<&str> { + let mut segments = request_path.splitn(5, '/'); + match ( + segments.next(), + segments.next(), + segments.next(), + segments.next(), + segments.next(), + ) { + (Some(""), Some("services"), Some(_), Some("api"), Some(path)) => Some(path), + _ => None, + } +} + +/// One route a model server answers, as the docs describe it. +struct RouteDoc { + method: HttpMethod, + path: &'static str, + summary: &'static str, + description: &'static str, + /// An example request body, built for the server's real model id. + example_body: Option Value>, +} + +/// The paths the docs describe for a server of `engine`, in order. `rocm`'s +/// tests compare them with what each engine reports as `supported_routes`. +#[must_use] +pub fn documented_route_paths(engine: &str) -> Vec { + routes_for(engine) + .iter() + .map(|route| route.path.to_owned()) + .collect() +} + +/// The routes each engine's server answers: the engine's health route from +/// [`rocm_engine_protocol::ENGINE_HEALTH_ROUTES`], when it has one there, plus +/// the chat routes. An engine the table does not know gets no health route +/// rather than another engine's. +fn routes_for(engine: &str) -> Vec { + let health = rocm_engine_protocol::engine_health_route(engine).map(|path| RouteDoc { + method: HttpMethod::Get, + path, + summary: "Check the server is up", + description: "Answers 200 once the server accepts requests.", + example_body: None, + }); + health.into_iter().chain(chat_routes()).collect() +} + +fn chat_routes() -> [RouteDoc; 3] { + [ + RouteDoc { + method: HttpMethod::Get, + path: "/v1/models", + summary: "List the served models", + description: "OpenAI-compatible model list. The `id` of each entry is the \ + `model` value requests use.", + example_body: None, + }, + RouteDoc { + method: HttpMethod::Post, + path: "/v1/chat/completions", + summary: "Chat completion", + description: "OpenAI-compatible chat completion. Set `stream` to `true` to \ + receive server-sent events as tokens are generated.", + example_body: Some(chat_completion_example), + }, + RouteDoc { + method: HttpMethod::Post, + path: "/v1/completions", + summary: "Text completion", + description: "OpenAI-compatible text completion of a raw prompt.", + example_body: Some(completion_example), + }, + ] +} + +fn chat_completion_example(model: &str) -> Value { + json!({ + "model": model, + "messages": [{ "role": "user", "content": "Say hello in one sentence." }], + "max_tokens": 64, + "stream": false, + }) +} + +fn completion_example(model: &str) -> Value { + json!({ + "model": model, + "prompt": "The capital of France is", + "max_tokens": 16, + }) +} + +/// Where each engine documents more than rocm-cli describes. +fn engine_docs_note(record: &ManagedServiceRecord) -> String { + if let Some(own_docs) = record.engine_docs_url() { + return format!( + "vLLM also documents every route it serves, beyond the ones listed here, at \ + [{own_docs}]({own_docs})." + ); + } + match record.engine.as_str() { + "lemonade" => "On Linux, Lemonade runs chat models with the llama.cpp server, which also \ + answers its own routes; see the \ + [llama.cpp server documentation](https://github.com/ggml-org/llama.cpp/tree/master/tools/server)." + .to_owned(), + "aim" => "This server runs in an AMD Inference Microservice (AIM) container; the routes \ + listed here are the OpenAI-compatible ones rocm-cli uses." + .to_owned(), + _ => "The routes listed here are the OpenAI-compatible ones rocm-cli uses.".to_owned(), + } +} + +fn service_openapi(record: &ManagedServiceRecord) -> OpenApi { + // The name the server answers to, which for AIM is not the image + // reference recorded as the canonical id. + let model = record.served_model_name(); + let mut paths = PathsBuilder::new(); + for route in routes_for(&record.engine) { + let mut operation = OperationBuilder::new() + .summary(Some(route.summary)) + .description(Some(route.description)) + .response( + "200", + ResponseBuilder::new() + .description("The server's answer") + .content("application/json", ContentBuilder::new().build()), + ); + if let Some(example_body) = route.example_body { + operation = operation.request_body(Some( + RequestBodyBuilder::new() + .required(Some(Required::True)) + .content( + "application/json", + ContentBuilder::new() + .example(Some(example_body(model))) + .build(), + ) + .build(), + )); + } + paths = paths.path(route.path, PathItem::new(route.method, operation)); + } + + let description = format!( + "`{model}` served by {engine} at `{endpoint}`.\n\n\ + \"Try it out\" sends requests through this page to the server.\n\n{note}", + engine = record.engine, + endpoint = record.endpoint_url, + note = engine_docs_note(record), + ); + let mut builder = OpenApiBuilder::new() + .info( + InfoBuilder::new() + .title(format!("{model} ({})", record.service_id)) + .version(env!("CARGO_PKG_VERSION")) + .description(Some(description)), + ) + .servers(Some([ServerBuilder::new() + .url(format!("{}/api", service_path(&record.service_id))) + .description(Some(format!( + "{} through the docs page", + record.endpoint_url + ))) + .build()])) + .paths(paths); + if record.requires_api_key { + builder = builder + .components(Some( + ComponentsBuilder::new() + .security_scheme( + ENDPOINT_KEY_SCHEME, + SecurityScheme::Http( + HttpBuilder::new().scheme(HttpAuthScheme::Bearer).build(), + ), + ) + .build(), + )) + .security(Some([SecurityRequirement::new( + ENDPOINT_KEY_SCHEME, + Vec::::new(), + )])); + } + builder.build() +} + +fn html_escape(text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + match character { + '&' => escaped.push_str("&"), + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + '\'' => escaped.push_str("'"), + other => escaped.push(other), + } + } + escaped +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::temp_app_paths; + use axum::Json; + use std::fs; + + /// Serializes the tests that set proxy variables. Every other test here + /// talks to its servers through [`test_client`], which ignores them. + static PROXY_ENV_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + /// The test's own HTTP client: never proxied, so a proxy variable set by + /// the proxy test cannot reroute it, and never following a redirect, so a + /// test sees exactly what the docs server answered. + fn test_client() -> reqwest::Client { + reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap() + } + + /// The records exactly as written, the loader the daemon's own code uses. + fn records_as_written() -> ServiceLoader { + Arc::new(|paths: &AppPaths| crate::persistence::load_managed_services(paths)) + } + + fn start(paths: &AppPaths) -> Result { + start_api_docs_server(paths.clone(), 0, records_as_written()) + } + + fn record(paths: &AppPaths, service_id: &str, engine: &str, port: u16) -> ManagedServiceRecord { + let mut record = ManagedServiceRecord::new( + paths, + service_id, + engine, + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record + } + + fn services_dir_ready(paths: &AppPaths) -> Result<()> { + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + Ok(()) + } + + fn operation_paths(openapi: &Value) -> Vec { + openapi["paths"] + .as_object() + .expect("paths object") + .keys() + .cloned() + .collect() + } + + #[test] + fn chat_server_docs_list_its_routes_with_the_served_model() { + let (root, paths) = temp_app_paths("api-docs-chat-routes"); + for (engine, health) in [ + ("vllm", "/health"), + ("lemonade", "/v1/health"), + ("aim", "/health"), + ] { + let mut served = record(&paths, "svc-1", engine, 9); + // As for an AIM server, whose canonical id is the image reference + // and whose served name is adopted from the engine. + served.canonical_model_id = "registry.example/aim/qwen:1.0".to_owned(); + served.served_model_id = Some("Qwen/Qwen3.5".to_owned()); + let openapi = serde_json::to_value(service_openapi(&served)).unwrap(); + assert!( + openapi["info"]["title"] + .as_str() + .is_some_and(|title| title.starts_with("Qwen/Qwen3.5 ")), + "{engine}: {}", + openapi["info"]["title"] + ); + let mut listed = operation_paths(&openapi); + listed.sort(); + let mut expected = vec![ + health.to_owned(), + "/v1/chat/completions".to_owned(), + "/v1/completions".to_owned(), + "/v1/models".to_owned(), + ]; + expected.sort(); + assert_eq!(listed, expected, "{engine}"); + assert_eq!( + openapi["paths"]["/v1/chat/completions"]["post"]["requestBody"]["content"]["application/json"] + ["example"]["model"], + "Qwen/Qwen3.5", + "{engine}: try-it must start from the model the server answers to" + ); + assert_eq!( + openapi["servers"][0]["url"], "/services/svc-1/api", + "{engine}" + ); + assert!( + openapi.get("security").is_none(), + "{engine}: loopback needs no key" + ); + } + fs::remove_dir_all(root).ok(); + } + + /// The docs' own route list against the protocol's table. That each + /// engine reports that table is `rocm`'s + /// `documented_routes_are_the_routes_each_engine_reports`, which can call + /// the engines. + #[test] + fn documented_routes_are_the_protocols_route_table() { + for engine in ["vllm", "lemonade", "aim", "unknown"] { + assert_eq!( + documented_route_paths(engine), + rocm_engine_protocol::engine_supported_routes(engine), + "{engine}" + ); + } + } + + #[test] + fn forwarded_paths_keep_their_encoding() { + assert_eq!( + raw_forwarded_path("/services/svc-1/api/v1/models/Qwen%2FQwen3.5%3Fx"), + Some("v1/models/Qwen%2FQwen3.5%3Fx") + ); + assert_eq!(raw_forwarded_path("/services/svc-1/api/"), Some("")); + assert_eq!(raw_forwarded_path("/services/svc-1/docs/x"), None); + } + + #[test] + fn service_paths_percent_encode_the_id() { + assert_eq!(service_path("svc-1.a_b~"), "/services/svc-1.a_b~"); + assert_eq!(service_path("svc a#1%?/"), "/services/svc%20a%231%25%3F%2F"); + assert_eq!(service_path("é"), "/services/%C3%A9"); + } + + #[test] + fn only_the_docs_servers_own_pages_may_send_browser_requests() { + let own: Vec = vec![ + "http://127.0.0.1:11430".to_owned(), + "http://localhost:11430".to_owned(), + ]; + for origin in ["http://127.0.0.1:11430", "http://localhost:11430"] { + assert!(origin_is_own(origin, &own), "{origin}"); + } + for origin in [ + // Another local web app, on this machine but not these docs. + "http://localhost:3000", + "http://127.0.0.1:11431", + "https://127.0.0.1:11430", + "https://evil.example", + "http://127.0.0.1.evil.example", + "http://127.0.0.1:11430/", + "null", + "", + ] { + assert!(!origin_is_own(origin, &own), "{origin}"); + } + for site in ["same-origin", "none"] { + assert!(fetch_site_is_own(site), "{site}"); + } + // `same-site` covers every port of 127.0.0.1, other local apps too. + for site in ["same-site", "cross-site", "", "SAME-ORIGIN"] { + assert!(!fetch_site_is_own(site), "{site}"); + } + } + + #[test] + fn the_bundled_swagger_ui_is_the_version_named_in_the_notices() { + let config = Arc::new(utoipa_swagger_ui::Config::new(["/openapi.json"])); + let bundle = utoipa_swagger_ui::serve("swagger-ui-bundle.js", config) + .unwrap() + .expect("the vendored Swagger UI ships its bundle"); + let bundle = String::from_utf8_lossy(&bundle.bytes); + let marker = "PACKAGE_VERSION:\""; + let start = bundle.find(marker).expect("the bundle states its version") + marker.len(); + let version: String = bundle[start..].chars().take_while(|c| *c != '"').collect(); + // Read, not `include_str!`: the file is not ASCII, and xtask's output + // contract rejects non-ASCII data compiled into a binary. + let notices = fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../THIRD_PARTY_NOTICES.txt" + )) + .expect("THIRD_PARTY_NOTICES.txt at the repo root"); + assert!( + notices.contains(&format!(" - Swagger UI {version} ")), + "THIRD_PARTY_NOTICES.txt must name the Swagger UI {version} that rocmd serves; \ + update the bundled-assets section of about.hbs and run `cargo xtask tpn`" + ); + } + + #[test] + fn a_taken_port_is_reported_and_port_zero_binds_a_free_one() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-ports"); + paths.ensure()?; + let taken = std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, 0))?; + let taken_port = taken.local_addr()?.port(); + let error = start_api_docs_server(paths.clone(), taken_port, records_as_written()) + .err() + .expect("a taken port cannot be served on"); + assert!( + format!("{error:#}").contains(&format!("127.0.0.1:{taken_port}")), + "{error:#}" + ); + let server = start(&paths)?; + assert!(!server.base_url.ends_with(":0"), "{}", server.base_url); + assert_eq!( + server.service_docs_url("svc-1"), + format!("{}/services/svc-1/docs/", server.base_url) + ); + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[test] + fn keyed_server_docs_ask_for_the_endpoint_key() { + let (root, paths) = temp_app_paths("api-docs-keyed"); + let mut keyed = record(&paths, "svc-1", "vllm", 9); + keyed.requires_api_key = true; + let openapi = serde_json::to_value(service_openapi(&keyed)).unwrap(); + assert_eq!( + openapi["components"]["securitySchemes"][ENDPOINT_KEY_SCHEME]["scheme"], + "bearer" + ); + assert!(openapi["security"][0].get(ENDPOINT_KEY_SCHEME).is_some()); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn engine_note_names_each_engines_own_docs() { + let (root, paths) = temp_app_paths("api-docs-note"); + let note = |engine| engine_docs_note(&record(&paths, "a", engine, 8000)); + assert!(note("vllm").contains("http://127.0.0.1:8000/docs")); + assert!(note("lemonade").contains("llama.cpp")); + for engine in ["aim", "unknown"] { + let text = note(engine); + assert!( + !text.contains("llama.cpp") && !text.contains("/docs"), + "{engine} is not Lemonade or vLLM: {text}" + ); + } + assert!(note("aim").contains("AIM")); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn only_loopback_host_names_are_accepted() { + for host in [ + "127.0.0.1:11430", + "localhost:11430", + "LOCALHOST", + "[::1]:11430", + "127.0.0.1", + ] { + assert!(host_header_is_loopback(host), "{host}"); + } + for host in [ + "evil.example:11430", + "192.168.1.5:11430", + "[2001:db8::1]:11430", + "", + ] { + assert!(!host_header_is_loopback(host), "{host}"); + } + } + + #[test] + fn html_escape_neutralises_markup() { + assert_eq!( + html_escape("'&'"), + "<a href="x">'&'</a>" + ); + } + + fn header_text(headers: &HeaderMap, name: header::HeaderName) -> Option { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + } + + /// What reached the stand-in model server, as it saw it: the path still + /// percent-encoded, as the request line carried it. + async fn echo(method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Json { + Json(json!({ + "method": method.as_str(), + "path": uri.path(), + "query": uri.query(), + "content_type": header_text(&headers, header::CONTENT_TYPE), + "accept": header_text(&headers, header::ACCEPT), + "authorization": header_text(&headers, header::AUTHORIZATION), + "cookie": header_text(&headers, header::COOKIE), + "body_len": body.len(), + "body": String::from_utf8_lossy(&body), + })) + } + + /// A stand-in model server on `host` that echoes what reached it, so a + /// test can see exactly what the forwarder passed on. `/redirect` + /// answers with a redirect. `None` when `host` cannot be bound here. + async fn start_echo_model_server_on(host: IpAddr) -> Option { + let listener = TcpListener::bind((host, 0)).await.ok()?; + let port = listener.local_addr().ok()?.port(); + let app = Router::new() + .route( + "/redirect", + any(|| async { + ( + StatusCode::TEMPORARY_REDIRECT, + [(header::LOCATION, "/v1/models")], + ) + }), + ) + .fallback(echo) + .layer(DefaultBodyLimit::disable()); + tokio::spawn(async move { axum::serve(listener, app).await.ok() }); + Some(port) + } + + async fn start_echo_model_server() -> u16 { + start_echo_model_server_on(IpAddr::from([127, 0, 0, 1])) + .await + .expect("bind 127.0.0.1") + } + + #[tokio::test(flavor = "multi_thread")] + async fn docs_server_describes_and_forwards_to_a_running_server() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-live"); + services_dir_ready(&paths)?; + let model_port = start_echo_model_server().await; + record(&paths, "svc-live", "vllm", model_port).write()?; + let mut stopped = record(&paths, "svc-stopped", "vllm", model_port); + stopped.status = "stopped".to_owned(); + stopped.write()?; + + let server = start(&paths)?; + let base = server.base_url.clone(); + let client = test_client(); + + let index = client.get(format!("{base}/")).send().await?.text().await?; + assert!(index.contains("/services/svc-live/docs/"), "{index}"); + assert!(!index.contains("svc-stopped"), "{index}"); + + let page = client + .get(format!("{base}/services/svc-live/docs/")) + .send() + .await?; + assert_eq!(page.status(), StatusCode::OK); + assert!(page.text().await?.contains("swagger-ui")); + let initializer = client + .get(format!( + "{base}/services/svc-live/docs/swagger-initializer.js" + )) + .send() + .await? + .text() + .await?; + assert!( + initializer.contains("/services/svc-live/openapi.json"), + "Swagger UI must load this server's description: {initializer}" + ); + assert!( + initializer.contains("\"persistAuthorization\": false"), + "the endpoint key must not be saved in the browser: {initializer}" + ); + + let openapi: Value = client + .get(format!("{base}/services/svc-live/openapi.json")) + .send() + .await? + .text() + .await? + .parse()?; + assert_eq!(openapi["servers"][0]["url"], "/services/svc-live/api"); + + let forwarded: Value = client + .post(format!( + "{base}/services/svc-live/api/v1/chat/completions?trace=1" + )) + .header(header::AUTHORIZATION, "Bearer typed-in-docs") + .header(header::COOKIE, "session=browser-only") + .header(header::CONTENT_TYPE, "application/json") + .header(header::ACCEPT, "text/event-stream") + .body(r#"{"model":"Qwen/Qwen3.5"}"#) + .send() + .await? + .text() + .await? + .parse()?; + assert_eq!(forwarded["method"], "POST"); + assert_eq!(forwarded["path"], "/v1/chat/completions"); + assert_eq!(forwarded["body"], r#"{"model":"Qwen/Qwen3.5"}"#); + assert_eq!(forwarded["query"], "trace=1"); + assert_eq!(forwarded["authorization"], "Bearer typed-in-docs"); + assert_eq!( + forwarded["content_type"], "application/json", + "the server parses the body by it" + ); + assert_eq!( + forwarded["accept"], "text/event-stream", + "a streamed answer is asked for by it" + ); + assert_eq!( + forwarded["cookie"], + Value::Null, + "browser cookies stay in the browser" + ); + + for path in ["docs/", "openapi.json", "api/v1/models"] { + let response = client + .get(format!("{base}/services/svc-stopped/{path}")) + .send() + .await?; + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{path}"); + } + let stopped_page = client + .get(format!("{base}/services/svc-stopped/docs/")) + .send() + .await? + .text() + .await?; + assert!(stopped_page.contains("is not running"), "{stopped_page}"); + + let rebound = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::HOST, "attacker.example") + .body("{}") + .send() + .await?; + assert_eq!(rebound.status(), StatusCode::FORBIDDEN, "DNS rebinding"); + let cross_site = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::ORIGIN, "https://some-site.example") + .header(header::CONTENT_TYPE, "text/plain") + .body("{}") + .send() + .await?; + assert_eq!( + cross_site.status(), + StatusCode::FORBIDDEN, + "cross-site post" + ); + // Another web app on this machine is still not these docs. + let other_local_app = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::ORIGIN, "http://localhost:3000") + .header(header::CONTENT_TYPE, "text/plain") + .body("{}") + .send() + .await?; + assert_eq!( + other_local_app.status(), + StatusCode::FORBIDDEN, + "a page from another local port" + ); + let localhost_alias = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header( + header::ORIGIN, + base.replace("127.0.0.1", "localhost").as_str(), + ) + .body("{}") + .send() + .await?; + assert_eq!( + localhost_alias.status(), + StatusCode::OK, + "the docs page opened as localhost" + ); + let same_origin = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::ORIGIN, base.as_str()) + .body("{}") + .send() + .await?; + assert_eq!(same_origin.status(), StatusCode::OK, "the docs page itself"); + + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// A browser sends no `Origin` with a link, an image or a navigation, but + /// it does send `Sec-Fetch-Site`; only this server's own pages and the + /// user's own typing get through. + #[tokio::test(flavor = "multi_thread")] + async fn origin_less_browser_requests_must_come_from_the_docs_pages() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-fetch-site"); + services_dir_ready(&paths)?; + let model_port = start_echo_model_server().await; + record(&paths, "svc-live", "vllm", model_port).write()?; + let server = start(&paths)?; + let url = format!("{}/services/svc-live/api/v1/models", server.base_url); + let client = test_client(); + + for (site, expected) in [ + (Some("cross-site"), StatusCode::FORBIDDEN), + (Some("same-site"), StatusCode::FORBIDDEN), + (Some("same-origin"), StatusCode::OK), + (Some("none"), StatusCode::OK), + // Not a browser: curl and other programs send neither header. + (None, StatusCode::OK), + ] { + let mut request = client.get(&url); + if let Some(site) = site { + request = request.header(SEC_FETCH_SITE, site); + } + assert_eq!(request.send().await?.status(), expected, "{site:?}"); + } + + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn no_other_site_can_frame_the_docs() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-frames"); + services_dir_ready(&paths)?; + let model_port = start_echo_model_server().await; + record(&paths, "svc-live", "vllm", model_port).write()?; + let server = start(&paths)?; + let base = server.base_url.clone(); + let client = test_client(); + + for (path, csp) in [ + ("/", PAGE_CSP), + ("/services/svc-live/docs/", PAGE_CSP), + ("/services/svc-live/docs/swagger-initializer.js", PAGE_CSP), + ("/services/svc-live/openapi.json", PAGE_CSP), + ("/services/not-running/docs/", PAGE_CSP), + ("/services/svc-live/api/v1/models", FORWARDED_CSP), + ] { + let response = client.get(format!("{base}{path}")).send().await?; + let headers = response.headers(); + assert_eq!(headers[header::CONTENT_SECURITY_POLICY], csp, "{path}"); + assert_eq!(headers[header::X_FRAME_OPTIONS], "DENY", "{path}"); + assert_eq!(headers[header::X_CONTENT_TYPE_OPTIONS], "nosniff", "{path}"); + } + let refused = client + .get(format!("{base}/")) + .header(header::HOST, "attacker.example") + .send() + .await?; + assert_eq!(refused.status(), StatusCode::FORBIDDEN); + assert_eq!(refused.headers()[header::X_FRAME_OPTIONS], "DENY"); + + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn try_it_keeps_the_path_encoded_and_sends_large_bodies() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-forward-path"); + services_dir_ready(&paths)?; + let model_port = start_echo_model_server().await; + record(&paths, "svc-live", "vllm", model_port).write()?; + let server = start(&paths)?; + let base = server.base_url.clone(); + let client = test_client(); + + // Decoded, `%2F` would add a segment and `%3F` would start a query. + let forwarded: Value = client + .get(format!( + "{base}/services/svc-live/api/v1/models/Qwen%2FQwen3.5%3Fx" + )) + .send() + .await? + .text() + .await? + .parse()?; + assert_eq!(forwarded["path"], "/v1/models/Qwen%2FQwen3.5%3Fx"); + assert_eq!(forwarded["query"], Value::Null); + + // Larger than axum's 2 MiB default, as an audio upload can be. + let large = 3 * 1024 * 1024; + let response = client + .post(format!( + "{base}/services/svc-live/api/v1/audio/transcriptions" + )) + .body(vec![b'a'; large]) + .send() + .await?; + assert_eq!(response.status(), StatusCode::OK); + let forwarded: Value = response.text().await?.parse()?; + assert_eq!(forwarded["body_len"], large); + + // A redirect is passed back, not followed by the forwarder. + let redirect = client + .get(format!("{base}/services/svc-live/api/redirect")) + .send() + .await?; + assert_eq!(redirect.status(), StatusCode::TEMPORARY_REDIRECT); + + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn ids_are_escaped_in_pages_and_encoded_in_links() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-escaping"); + services_dir_ready(&paths)?; + let model_port = start_echo_model_server().await; + let odd_id = "svc a#1%"; + let mut odd = record(&paths, odd_id, "vllm", model_port); + odd.canonical_model_id = "model".to_owned(); + odd.write()?; + let server = start(&paths)?; + let base = server.base_url.clone(); + let client = test_client(); + + let link = server.service_docs_url(odd_id); + assert_eq!(link, format!("{base}/services/svc%20a%231%25/docs/")); + let page = client.get(&link).send().await?; + assert_eq!(page.status(), StatusCode::OK, "the printed link opens"); + let initializer = client + .get(format!("{link}swagger-initializer.js")) + .send() + .await? + .text() + .await?; + assert!( + initializer.contains("/services/svc%20a%231%25/openapi.json"), + "{initializer}" + ); + let openapi = client + .get(format!("{base}/services/svc%20a%231%25/openapi.json")) + .send() + .await?; + assert_eq!(openapi.status(), StatusCode::OK); + let openapi: Value = openapi.text().await?.parse()?; + assert_eq!(openapi["servers"][0]["url"], "/services/svc%20a%231%25/api"); + let redirect = client + .get(format!("{base}/services/svc%20a%231%25/docs")) + .send() + .await?; + assert_eq!( + redirect.headers()[header::LOCATION], + "/services/svc%20a%231%25/docs/" + ); + + let index = client.get(format!("{base}/")).send().await?.text().await?; + assert!( + index.contains("href=\"/services/svc%20a%231%25/docs/\""), + "{index}" + ); + assert!(index.contains("<b>model</b>"), "{index}"); + assert!(!index.contains(""), "{index}"); + + // The id of a server that is not running is reflected into the page. + let crafted = client + .get(format!( + "{base}/services/%3Cscript%3Ealert(1)%3C%2Fscript%3E/docs/" + )) + .send() + .await?; + assert_eq!(crafted.status(), StatusCode::NOT_FOUND); + let crafted = crafted.text().await?; + assert!(crafted.contains("<script>alert(1)"), "{crafted}"); + assert!(!crafted.contains("