From 588a3c685be754e54ba8a680fce84b6b1760d6d0 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 9 Nov 2026 16:38:19 +0200 Subject: [PATCH 1/2] feat: serve API docs for running model servers with `rocm services docs` `rocm services docs []` serves browsable API docs (Swagger UI) for the running model servers on 127.0.0.1, in the foreground, until Ctrl+C. Each server's page lists the routes it answers, with request examples filled in with the served model, and its "Try it out" forwards requests through the page to the server, so it works for every engine whatever the engine's own cross-origin settings. Examples, titles and labels use the name the server answers to, which for AIM is not the image reference. The command refuses, before serving anything, when the named server is not running (naming `rocm services restart --yes`) or when no server is. It uses port 11430, or a free port when 11430 is in use and says so on stderr; `--port` picks one and never falls back. Nothing runs in the background: after Ctrl+C nothing is left serving and the model servers are not touched. The docs judge which servers are running with the loader `rocm services list` uses, which checks each server's process and endpoint, so a server that stops or crashes while the docs are open drops off them about a second after the read that finds it gone. Requests that arrive together share one read. `rocm serve` (summary, plain and attached output) and `rocm services list` print a `docs:` line naming the command for every running server, and an `engine docs:` line with vLLM's own /docs page. The routes the docs describe come from one table in rocm-engine-protocol: the OpenAI chat routes plus each engine's health route. The engines report the same table as `supported_routes` in their `endpoint` answer, which a test in `rocm` asks each engine for and compares with the docs. The forwarder passes on only Content-Type, Accept and Authorization, keeps the request path percent-encoded, accepts bodies up to 64 MiB, does not follow redirects, gives up on a server that does not accept the connection within 5 s, and bypasses any configured proxy. It reaches a server bound to `0.0.0.0` on 127.0.0.1 and one bound to `::` on ::1, so an IPv6-only socket is reached too; that mapping is now one helper in rocm-core (`reachable_host`), which the Anthropic proxy uses as well. Browser requests are admitted only from the docs server's own pages: by `Origin` when the browser sends one, and by `Sec-Fetch-Site` when it does not. The `Host` guard keeps DNS rebinding out. Every answer carries `frame-ancestors 'none'`, `X-Frame-Options: DENY` and nosniff, and forwarded answers are also CSP-sandboxed. Server ids are percent-encoded in links and escaped in pages. Swagger UI is vendored (no CDN); its licence is in the notices. Scenarios service-docs-01 to -11 cover the listed command opening the docs, try-it, a server stopped while its docs are open, refusals made before any port is used, Ctrl+C stopping the docs while the model server keeps answering, the fallback from a taken port 11430, the `engine docs` link, the advice given for a stopped server, and the rows `rocm serve` prints when the model is already running, piped and attached. Signed-off-by: Roman Inflianskas --- Cargo.lock | 68 +- MANIFEST.md | 13 + README.md | 46 +- THIRD_PARTY_NOTICES.txt | 339 +++ about.hbs | 127 ++ apps/rocm/src/cli.rs | 43 +- apps/rocm/src/main.rs | 1 + apps/rocm/src/providers.rs | 1 + apps/rocm/src/serve_cmd.rs | 7 +- apps/rocm/src/serve_summary.rs | 41 + apps/rocm/src/service_launch/attach.rs | 11 + apps/rocm/src/service_launch/start.rs | 8 + apps/rocm/src/services_cmd.rs | 6 +- apps/rocm/src/services_docs.rs | 459 ++++ apps/rocm/src/services_text.rs | 69 +- apps/rocmd/Cargo.toml | 10 + apps/rocmd/src/anthropic_proxy.rs | 11 +- apps/rocmd/src/api_docs.rs | 1955 +++++++++++++++++ apps/rocmd/src/lib.rs | 5 + crates/e2e-report/src/command_coverage.rs | 1 + crates/rocm-core/src/lib.rs | 2 +- crates/rocm-core/src/net.rs | 32 + crates/rocm-core/src/service_record.rs | 68 + crates/rocm-engine-protocol/src/lib.rs | 67 + docs/architecture.md | 17 +- docs/manual-testing.md | 36 + docs/testing.md | 75 + engines/aim/src/lib.rs | 7 + engines/aim/src/state.rs | 7 +- engines/lemonade/src/lib.rs | 14 +- engines/vllm/src/lib.rs | 7 + engines/vllm/src/state.rs | 7 +- tests/e2e-cucumber/Cargo.toml | 8 +- .../features/model_serving.feature | 3 + .../features/service_docs.feature | 114 + tests/e2e-cucumber/src/mock_server.rs | 13 +- tests/e2e-cucumber/tests/e2e.rs | 20 + .../tests/e2e/service_docs_steps.rs | 789 +++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 4 +- tests/e2e-cucumber/tests/feature_naming.rs | 1 + 40 files changed, 4465 insertions(+), 47 deletions(-) create mode 100644 apps/rocm/src/services_docs.rs create mode 100644 apps/rocmd/src/api_docs.rs create mode 100644 tests/e2e-cucumber/features/service_docs.feature create mode 100644 tests/e2e-cucumber/tests/e2e/service_docs_steps.rs diff --git a/Cargo.lock b/Cargo.lock index ed1f160cf0..1e26e3f264 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2118,6 +2118,8 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -3590,6 +3592,7 @@ dependencies = [ "base64 0.22.1", "bytes", "futures-core", + "futures-util", "http", "http-body", "http-body-util", @@ -3604,12 +3607,14 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-util", "tower", "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.4.2", "web-sys", ] @@ -3653,7 +3658,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.5.0", "web-sys", ] @@ -3935,6 +3940,7 @@ dependencies = [ "axum", "clap", "libc", + "reqwest 0.12.28", "rocm-core", "rocm-engine-protocol", "serde", @@ -3942,6 +3948,8 @@ dependencies = [ "sha2", "tokio", "ureq", + "utoipa", + "utoipa-swagger-ui", ] [[package]] @@ -5452,6 +5460,51 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "utoipa" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8765fe27aeff71012a3f90fa474cf1df863a7d0dd81ed25de4e63fff2544994f" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d935f1c83fdf8b88f09bbe050d0cea78ea4afee6d7b0317403228c1200b2c8ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "10.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1fd2f9615838885c6ff4115fde8a2b58825dc7b4617f78d29c167377a15c92" +dependencies = [ + "base64 0.23.1", + "mime_guess", + "regex", + "serde", + "serde_json", + "utoipa", + "utoipa-swagger-ui-vendored", + "zip", +] + +[[package]] +name = "utoipa-swagger-ui-vendored" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552a64d6a0d251ae28bc021838800943cce60c9ec2e60ff331f54e79f696268f" + [[package]] name = "uuid" version = "1.23.3" @@ -5596,6 +5649,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasm-streams" version = "0.5.0" diff --git a/MANIFEST.md b/MANIFEST.md index ca33e22907..5acc20a085 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -538,6 +538,10 @@ repository. | utf-8 | 0.7.6 | MIT OR Apache-2.0 | | utf8_iter | 1.0.4 | Apache-2.0 OR MIT | | utf8parse | 0.2.2 | Apache-2.0 OR MIT | +| utoipa | 6.0.0 | MIT OR Apache-2.0 | +| utoipa-gen | 6.0.1 | MIT OR Apache-2.0 | +| utoipa-swagger-ui | 10.0.2 | MIT OR Apache-2.0 | +| utoipa-swagger-ui-vendored | 0.2.0 | MIT OR Apache-2.0 | | uuid | 1.23.3 | Apache-2.0 OR MIT | | valuable | 0.1.1 | MIT | | version_check | 0.9.5 | MIT/Apache-2.0 | @@ -553,6 +557,7 @@ repository. | wasm-bindgen-macro | 0.2.125 | MIT OR Apache-2.0 | | wasm-bindgen-macro-support | 0.2.125 | MIT OR Apache-2.0 | | wasm-bindgen-shared | 0.2.125 | MIT OR Apache-2.0 | +| wasm-streams | 0.4.2 | MIT OR Apache-2.0 | | wasm-streams | 0.5.0 | MIT OR Apache-2.0 | | web-sys | 0.3.102 | MIT OR Apache-2.0 | | web-time | 1.1.0 | MIT OR Apache-2.0 | @@ -775,6 +780,14 @@ With the prek hooks installed this runs automatically whenever you commit a dependency-graph change, so the notices rarely go stale by hand; the hook skips if cargo-about is absent or not the pinned version. +Assets compiled in that are not crates — today, the Swagger UI bundle behind +`rocm services docs` — are listed in a "Bundled web assets" section maintained +by hand at the end of `about.hbs`. `cargo-about` cannot see them, so their +licenses bypass the `about.toml` check: when a dependency bump changes such an +asset, update that section from the asset's own upstream notices. A test in +`rocmd` fails when the served Swagger UI version is not the one the notices +name. + CI verifies the committed file is current with `cargo xtask tpn --check`; a new dependency under a license not listed in `about.toml` fails the check until the license is reviewed and added (or the dependency dropped). CI pins the diff --git a/README.md b/README.md index 09dca9adf9..6633f470a7 100644 --- a/README.md +++ b/README.md @@ -1251,8 +1251,9 @@ the running service, or stop it first to serve on the new port. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table -with the status, the full inference endpoint, the API-qualified model name, and a -quick smoke test (time to first token and approximate tokens per second). Control +with the status, the engine, the API-qualified model name, the full inference +endpoint, the command that serves the server's API docs (see API docs below), +and a quick smoke test (time to first token and approximate tokens per second). Control returns to your shell with the server still running; manage it later with `rocm services` (below). @@ -1477,6 +1478,7 @@ Manage background servers started with `--managed`: ``` rocm services list [--all] [--json] +rocm services docs [] [--port ] rocm services logs [--follow] rocm services stop --yes rocm services restart --yes @@ -1549,6 +1551,46 @@ started while a `prune` is scanning waits for the prune. `--json` prints the service records verbatim, for scripting and for the remote orchestration below. +#### API docs + +`rocm services docs` serves interactive API docs for the running model servers, +in the foreground, until you press Ctrl+C. `rocm serve` names the command next +to the endpoint, and `rocm services list` repeats it for each running server +(`--json` prints the records alone, without it): + +``` + endpoint http://127.0.0.1:11435/v1/chat/completions + docs rocm services docs +``` + +Run it to get the links: + +``` +$ rocm services docs +API docs for running model servers: http://127.0.0.1:11430/ + (): http://127.0.0.1:11430/services//docs/ +Press Ctrl+C to stop serving the docs; the model servers keep running. +``` + +Without a service id it links every running server. It refuses, and serves +nothing, when the named server is not running or when no server is. + +The page lists the routes the server answers (the model list, chat and text +completions, and the engine's health check), with request examples already +filled in with the served model. "Try it out" sends the request through the +docs page to the server, so it works for every engine without enabling +cross-origin access on the server; for a server started with an endpoint key, +enter the key under "Authorize" (it is kept only until the page is closed). +The root page lists every running server, judged as `rocm services list` judges +it; a server that stops or exits drops off about a second later. vLLM also documents its full API at +its own `/docs` page: `rocm serve` and `rocm services list` print it as +`engine docs`, and the docs page links to it. + +The docs listen on 127.0.0.1 only, answer browser requests only from their own +pages, cannot be framed by other sites, and work offline. They use port 11430, or a free port when 11430 is +taken; `--port ` picks one and fails if it is taken. Nothing keeps +running after Ctrl+C, and the model servers are never touched. + ### Remote machines (preview) Run a model on a different GPU machine and reach it from your own. Both machines diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index 175271e8c7..d2528a21ab 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -1934,6 +1934,217 @@ The following component(s) are licensed under Apache License 2.0: Apache License 2.0 ================================================================================ +The following component(s) are licensed under Apache License 2.0: + - utoipa-gen 6.0.1 + - utoipa-swagger-ui 10.0.2 + - utoipa 6.0.0 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================================ +Apache License 2.0 +================================================================================ + The following component(s) are licensed under Apache License 2.0: - rustls-platform-verifier 0.7.0 - unicode-linebreak 0.1.5 @@ -8421,6 +8632,7 @@ The following component(s) are licensed under Apache License 2.0: - unicode-ident 1.0.24 - utf8parse 0.2.2 - vte 0.15.0 + - wasm-streams 0.4.2 - wasm-streams 0.5.0 - winapi-i686-pc-windows-gnu 0.4.0 - winapi-x86_64-pc-windows-gnu 0.4.0 @@ -12124,3 +12336,130 @@ the following restrictions: 3. This notice may not be removed or altered from any source distribution. +================================================================================ +Bundled web assets +================================================================================ + +These components are not Rust crates; they are compiled into rocm-cli by a +crate listed above. Each license's full text appears above. + + - Swagger UI 5.32.6 + Licensed under Apache License 2.0. Compiled in by utoipa-swagger-ui + (through utoipa-swagger-ui-vendored) for the API docs of running model + servers. Its NOTICE file reads: + + swagger-ui + Copyright 2020-2021 SmartBear Software Inc. + + Swagger UI's bundle also contains third-party JavaScript. The notices it + retains, as distributed with Swagger UI 5.32.6 + (swagger-ui-bundle.js.LICENSE.txt in the swagger-ui-dist package; the + standalone preset's notices are a subset of these), read: + + /*! + Copyright (c) 2018 Jed Watson. + Licensed under the MIT License (MIT), see + http://jedwatson.github.io/classnames + */ + + /*! + * @description Recursive object extending + * @author Viacheslav Lotsmanov + * @license MIT + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Viacheslav Lotsmanov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + + /*! + * The buffer module from node.js, for the browser. + * + * @author Feross Aboukhadijeh + * @license MIT + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2021 Joachim Wester + * MIT license + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2022 Joachim Wester + * MIT licensed + */ + + /*! + * repeat-string + * + * Copyright (c) 2014-2015, Jon Schlinkert. + * Licensed under the MIT License. + */ + + /*! @license DOMPurify 3.4.0 | (c) Cure53 and other contributors | Released under the Apache license 2.0 and Mozilla Public License 2.0 | github.com/cure53/DOMPurify/blob/3.4.0/LICENSE */ + + /*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh */ + + /*! js-yaml 4.1.1 https://github.com/nodeca/js-yaml @license MIT */ + + /*! safe-buffer. MIT License. Feross Aboukhadijeh */ + + /** + * @license React + * react-dom.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * react.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * scheduler.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * use-sync-external-store-with-selector.production.js + * + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + diff --git a/about.hbs b/about.hbs index 0781534598..bf281fc8a8 100644 --- a/about.hbs +++ b/about.hbs @@ -30,3 +30,130 @@ The following component(s) are licensed under {{{name}}}: {{{text}}} {{/each}} + +================================================================================ +Bundled web assets +================================================================================ + +These components are not Rust crates; they are compiled into rocm-cli by a +crate listed above. Each license's full text appears above. + + - Swagger UI 5.32.6 + Licensed under Apache License 2.0. Compiled in by utoipa-swagger-ui + (through utoipa-swagger-ui-vendored) for the API docs of running model + servers. Its NOTICE file reads: + + swagger-ui + Copyright 2020-2021 SmartBear Software Inc. + + Swagger UI's bundle also contains third-party JavaScript. The notices it + retains, as distributed with Swagger UI 5.32.6 + (swagger-ui-bundle.js.LICENSE.txt in the swagger-ui-dist package; the + standalone preset's notices are a subset of these), read: + + /*! + Copyright (c) 2018 Jed Watson. + Licensed under the MIT License (MIT), see + http://jedwatson.github.io/classnames + */ + + /*! + * @description Recursive object extending + * @author Viacheslav Lotsmanov + * @license MIT + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Viacheslav Lotsmanov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + + /*! + * The buffer module from node.js, for the browser. + * + * @author Feross Aboukhadijeh + * @license MIT + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2021 Joachim Wester + * MIT license + */ + + /*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017-2022 Joachim Wester + * MIT licensed + */ + + /*! + * repeat-string + * + * Copyright (c) 2014-2015, Jon Schlinkert. + * Licensed under the MIT License. + */ + + /*! @license DOMPurify 3.4.0 | (c) Cure53 and other contributors | Released under the Apache license 2.0 and Mozilla Public License 2.0 | github.com/cure53/DOMPurify/blob/3.4.0/LICENSE */ + + /*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh */ + + /*! js-yaml 4.1.1 https://github.com/nodeca/js-yaml @license MIT */ + + /*! safe-buffer. MIT License. Feross Aboukhadijeh */ + + /** + * @license React + * react-dom.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * react.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * scheduler.production.min.js + * + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + + /** + * @license React + * use-sync-external-store-with-selector.production.js + * + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 0096036fed..9a0fec4ef4 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -460,7 +460,8 @@ rocm model --verbose" /// Anthropic `/v1/messages` endpoint is published beside the OpenAI /// one (see `rocm services list`). By default the /// server runs as a managed background service and prints a deployment summary - /// (status, endpoint, model, and smoke-test metrics); use --verbose to stream engine + /// (status, endpoint, the `rocm services docs` command for its API docs, model, and + /// smoke-test metrics); use --verbose to stream engine /// logs in this terminal instead. When streaming, press Ctrl-D to detach and leave the /// server running, or Ctrl-C to stop it. If that stop answers that it could not confirm /// the server exited, it says so on stderr and exits with an error; if the stop itself @@ -1177,6 +1178,10 @@ pub(crate) enum ComfyuiCommand { pub(crate) enum ServicesCommand { /// Show currently running local model servers, and count the records that /// are no longer running. + /// + /// Each running server gets a `docs:` line naming the command that serves + /// its API docs (`rocm services docs `); a vLLM server also gets an + /// `engine docs:` line, the docs vLLM serves itself. List { /// Include failed, stopped, and old service records. #[arg(short, long)] @@ -1189,6 +1194,23 @@ pub(crate) enum ServicesCommand { #[arg(long)] json: bool, }, + /// Serve browsable API docs for running local model servers until Ctrl+C. + /// + /// Serves a loopback-only page (Swagger UI) per running server listing the + /// routes it answers, with a working "Try it out" that sends requests + /// through the page to the server. Prints the page links, then serves them + /// until Ctrl+C; nothing keeps running afterwards and the model servers + /// are not touched. Refuses when the named server is not running, or when + /// no server is. + Docs { + /// Service id from `rocm services list`; without it, every running + /// server's docs are linked. + service_id: Option, + /// Port to serve the docs on (127.0.0.1 only). Without it, 11430 is + /// used, or a free port when 11430 is taken. + #[arg(long)] + port: Option, + }, /// Show logs for a local model server. Logs { /// Service id from `rocm services list`. @@ -1437,6 +1459,25 @@ mod tests { } } + /// The API docs and each engine's `supported_routes` take an engine's + /// health route from one table; an engine `rocm serve` accepts but the + /// table lacks would get docs without a health route. + #[test] + fn every_supported_engine_has_a_health_route_and_no_other_engine_does() { + for engine in SUPPORTED_ENGINES { + assert!( + rocm_engine_protocol::engine_health_route(engine).is_some(), + "{engine} is missing from ENGINE_HEALTH_ROUTES" + ); + } + for (engine, _) in rocm_engine_protocol::ENGINE_HEALTH_ROUTES { + assert!( + SUPPORTED_ENGINES.contains(&engine), + "ENGINE_HEALTH_ROUTES names {engine}, which `rocm serve` does not accept" + ); + } + } + /// EAI-7362: the top-level `rocm --help` command list must be alphabetical, /// including clap's implicit `help` entry (it must sort into place, not be /// pinned last). diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index f3fc7f3faf..4c59bf1a92 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -68,6 +68,7 @@ mod service_hints; mod service_launch; mod service_records; mod services_cmd; +mod services_docs; mod services_text; mod setup_cmd; mod shell_quote; diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index d35d738698..5306baf002 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -1925,6 +1925,7 @@ mod tests { metrics, api_key: None, notes: Vec::new(), + docs_rows: Vec::new(), }); assert!(rendered.contains("time to first token n/a")); assert!(rendered.contains("throughput (approx) n/a")); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 2336899be7..c6d01925b2 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -1316,6 +1316,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { metrics, api_key: launched_key, notes, + docs_rows: crate::services_docs::launch_docs_rows(&paths, &report.service_id), }; // Same intentional one-time key display as `print_managed_launch_plain` // in service_launch/start.rs; see its rationale for why the tag below is @@ -1324,7 +1325,11 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { print!("{}", serve_summary::render_summary(&summary)); } else { spinner.clear(); - print_managed_launch_plain(&report, launched_key.as_deref()); + print_managed_launch_plain( + &report, + launched_key.as_deref(), + &crate::services_docs::launch_docs_rows(&paths, &report.service_id), + ); } return Ok(()); } diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index cd0ab34bc6..a1a8fbc9dc 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -68,6 +68,10 @@ pub(crate) struct DeploymentSummary { pub api_key: Option, /// GPU/device warnings folded in from the serve plan. pub notes: Vec, + /// The `docs` row naming `rocm services docs`, and an `engine docs` row + /// for an engine that serves docs itself, from + /// [`crate::services_docs::launch_docs_rows`]; shown under the endpoints. + pub docs_rows: Vec<(&'static str, String)>, } /// Render `Some(ttft)` as a human duration, or `n/a` when the probe did not @@ -138,6 +142,7 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { model_rows, vec![("endpoint", summary.chat_endpoint.clone())], anthropic_rows, + summary.docs_rows.clone(), vec![ ("time to first token", format_ttft(summary.metrics.ttft)), ("throughput (approx)", format_tps(summary.metrics.gen_tps)), @@ -468,9 +473,45 @@ mod tests { }, api_key: None, notes: Vec::new(), + docs_rows: Vec::new(), } } + // Pins where the rows go. Which rows a launched server gets is + // `services_docs::launch_docs_rows`; that the `docs` command serves the + // docs is scenario service-docs-01, and that the line `rocm serve` prints + // does is the GPU serve canaries' last step (serve-05, serve-07). + #[test] + fn summary_lists_the_docs_rows_right_after_the_endpoint() { + let summary = DeploymentSummary { + docs_rows: vec![ + ("docs", "rocm services docs vllm-qwen-1720000000".to_owned()), + ("engine docs", "http://127.0.0.1:1337/docs".to_owned()), + ], + ..base_summary() + }; + let rendered = render_summary(&summary); + let lines: Vec<&str> = rendered.lines().map(str::trim).collect(); + let endpoint_row = lines + .iter() + .position(|line| line.starts_with("endpoint ")) + .expect("endpoint row"); + assert_eq!( + &lines[endpoint_row + 1..endpoint_row + 3], + [ + "docs rocm services docs vllm-qwen-1720000000", + "engine docs http://127.0.0.1:1337/docs", + ], + "{rendered}" + ); + assert!( + !render_summary(&base_summary()) + .lines() + .any(|line| line.trim_start().starts_with("docs ")), + "no docs rows, no docs line" + ); + } + #[test] fn format_ttft_uses_ms_under_a_second_and_seconds_above() { assert_eq!(format_ttft(Some(Duration::from_millis(180))), "180 ms"); diff --git a/apps/rocm/src/service_launch/attach.rs b/apps/rocm/src/service_launch/attach.rs index d76a90e8d4..4003f39195 100644 --- a/apps/rocm/src/service_launch/attach.rs +++ b/apps/rocm/src/service_launch/attach.rs @@ -104,6 +104,10 @@ pub(crate) fn run_attached_service( println!("model already being served"); println!(" service_id: {}", report.service_id); println!(" endpoint: {}", report.endpoint_url); + for (label, value) in crate::services_docs::launch_docs_rows(paths, &report.service_id) + { + println!(" {label}: {value}"); + } println!(" status: {}", report.status); println!(" logs: rocm logs {}", report.service_id); println!( @@ -138,8 +142,12 @@ pub(crate) fn run_attached_service( // printed as the "serve plan" block in `serve()`; extend it with the launch // coordinates and the streaming hint rather than repeating it. let endpoint = format!("{}/v1", format_http_base_url(host, port)); + let docs_rows = crate::services_docs::launch_docs_rows(paths, &service_id); println!(" service_id: {service_id}"); println!(" endpoint: {endpoint}"); + for (label, value) in &docs_rows { + println!(" {label}: {value}"); + } if let Some(key) = endpoint_api_key { // Same intentional one-time key display as `print_managed_launch_plain` // in `service_launch/start.rs`; see its rationale for why the tag below is @@ -168,6 +176,9 @@ pub(crate) fn run_attached_service( println!("detached - server still running"); println!(" service_id: {service_id}"); println!(" endpoint: {endpoint}"); + for (label, value) in &docs_rows { + println!(" {label}: {value}"); + } println!(" list: rocm services"); println!(" logs: rocm logs {service_id}"); println!(" stop: {}", service_hints::service_stop_hint(&service_id)); diff --git a/apps/rocm/src/service_launch/start.rs b/apps/rocm/src/service_launch/start.rs index 06cb93697e..ff9d27d181 100644 --- a/apps/rocm/src/service_launch/start.rs +++ b/apps/rocm/src/service_launch/start.rs @@ -177,11 +177,18 @@ pub(crate) fn start_managed_service( pub(crate) fn print_managed_launch_plain( report: &ManagedLaunchReport, endpoint_api_key: Option<&str>, + docs_rows: &[(&str, String)], ) { + let print_docs_rows = || { + for (label, value) in docs_rows { + println!(" {label}: {value}"); + } + }; if report.already_running { println!("managed service already running"); println!(" service_id: {}", report.service_id); println!(" endpoint: {}", report.endpoint_url); + print_docs_rows(); println!(" status: {}", report.status); println!(" note: existing service detected; no second process spawned"); return; @@ -192,6 +199,7 @@ pub(crate) fn print_managed_launch_plain( println!(" process_pid: {child_pid}"); } println!(" endpoint: {}", report.endpoint_url); + print_docs_rows(); if let Some(key) = endpoint_api_key { // Intentional one-time display of a freshly generated API key to the // terminal so the user can copy it — the designed delivery channel diff --git a/apps/rocm/src/services_cmd.rs b/apps/rocm/src/services_cmd.rs index f2cbc52623..4cbcb68be2 100644 --- a/apps/rocm/src/services_cmd.rs +++ b/apps/rocm/src/services_cmd.rs @@ -3,7 +3,8 @@ // SPDX-License-Identifier: MIT //! `rocm services`: the command handler (`services`), which dispatches the -//! `list`, `logs`, `stop`, `restart`, `remove` and `prune` subcommands, and +//! `list`, `docs` (to `services_docs.rs`), `logs`, `stop`, `restart`, +//! `remove` and `prune` subcommands, and //! `run_approved_service_action`, the approval-gated stop/restart path, with //! `finish_service_action`, which renders its result and fails a stop that //! could not confirm the server gone. @@ -60,6 +61,9 @@ pub(crate) fn services(command: Option) -> Result<()> { } Ok(()) } + ServicesCommand::Docs { service_id, port } => { + crate::services_docs::run(&paths, service_id.as_deref(), port) + } ServicesCommand::Logs { service_id, follow } => { if follow { follow_service_logs(&paths, &service_id) diff --git a/apps/rocm/src/services_docs.rs b/apps/rocm/src/services_docs.rs new file mode 100644 index 0000000000..dbb6328ae9 --- /dev/null +++ b/apps/rocm/src/services_docs.rs @@ -0,0 +1,459 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! `rocm services docs []`: browsable API docs for the running model +//! servers, served on loopback by this process until Ctrl+C. +//! +//! The docs server itself is `rocmd`'s [`rocmd::start_api_docs_server`], run +//! in-process. It runs only while this command does: nothing is left behind +//! when it stops, and the model servers are not touched. It is given the +//! loader `rocm services list` uses, so the docs list the same servers as +//! running that the list does, and a server that stops or crashes while the +//! docs are open drops off them. +//! +//! [`docs_hint`] is the line `rocm serve` and `rocm services list` print for a +//! running server, naming this command; the `engine docs` line beside it is +//! [`ManagedServiceRecord::engine_docs_url`], the docs an engine serves itself +//! (vLLM's `/docs`). + +use std::io::Write as _; + +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, ManagedServiceRecord}; + +use crate::managed_service_status::load_managed_services; +use crate::services_text::validate_service_id; + +/// The command that serves `service_id`'s API docs, as printed after a +/// running server's endpoint. The same string is what scenario +/// `service-docs-01` runs, so the hint and the command cannot drift apart. +pub(crate) fn docs_hint(service_id: &str) -> String { + format!( + "rocm services docs {}", + crate::shell_quote::shell_quote(service_id) + ) +} + +/// The rows `rocm serve` prints under a server's endpoint: `docs`, naming +/// this command, and `engine docs` when the engine serves docs of its own. +/// The record is read for its engine and address; one that cannot be read +/// still gets the `docs` row, since the command reports that itself. +pub(crate) fn launch_docs_rows(paths: &AppPaths, service_id: &str) -> Vec<(&'static str, String)> { + let mut rows = vec![("docs", docs_hint(service_id))]; + if let Some(engine_docs) = ManagedServiceRecord::read_manifest(paths, service_id) + .ok() + .as_ref() + .and_then(ManagedServiceRecord::engine_docs_url) + { + rows.push(("engine docs", engine_docs)); + } + rows +} + +/// `rocm services docs []`. +pub(crate) fn run(paths: &AppPaths, service_id: Option<&str>, port: Option) -> Result<()> { + if let Some(service_id) = service_id { + validate_service_id(service_id)?; + } + let records = load_managed_services(paths)?; + let targets = docs_targets(&records, service_id)?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("failed to build the runtime that waits for Ctrl+C")?; + // Listen for Ctrl+C before printing the link: the handler is installed + // when the future is first polled, and a Ctrl+C before that would kill + // the command without its closing line. One poll here installs it. + let mut interrupted = std::pin::pin!(tokio::signal::ctrl_c()); + let early = runtime.block_on(async { + tokio::select! { + biased; + result = &mut interrupted => Some(result), + () = std::future::ready(()) => None, + } + }); + // Defensive: Ctrl+C has to land within this first poll, which no test + // can aim at. A Ctrl+C after it, during startup, is caught by the handler + // and ends the wait below once the links are printed. + if let Some(result) = early { + // Nothing has been served or printed, so there is nothing to report. + result.context("failed to wait for Ctrl+C")?; + return Ok(()); + } + let (server, fallback_note) = + start_docs(paths, port, rocmd::DEFAULT_API_DOCS_PORT, |paths, port| { + rocmd::start_api_docs_server(paths, port, services_loader()) + })?; + + // On stderr, so stdout carries only the command's own output (the links + // and how to stop), which a caller reads from a pipe. + if let Some(note) = fallback_note { + eprintln!("{note}"); + } + let mut out = std::io::stdout().lock(); + writeln!( + out, + "API docs for running model servers: {}/", + server.base_url + )?; + for record in &targets { + writeln!( + out, + " {} ({}): {}", + record.service_id, + record.served_model_name(), + server.service_docs_url(&record.service_id) + )?; + } + writeln!( + out, + "Press Ctrl+C to stop serving the docs; the model servers keep running." + )?; + // A caller reading the link from a pipe must see it now, not at exit. + out.flush().context("failed to write the API docs link")?; + drop(out); + + let waited = runtime + .block_on(&mut interrupted) + .context("failed to wait for Ctrl+C"); + server.stop(); + waited?; + println!("API docs stopped."); + Ok(()) +} + +/// The loader the docs server reads the running servers with: the one +/// `rocm services list` uses, which checks each server's process and +/// endpoint. +fn services_loader() -> rocmd::ServiceLoader { + std::sync::Arc::new(|paths: &AppPaths| load_managed_services(paths)) +} + +/// The running servers whose docs links are printed: the one named, or every +/// running server. Refuses when there is nothing to document, so the command +/// never serves an empty page silently. +fn docs_targets<'a>( + records: &'a [ManagedServiceRecord], + service_id: Option<&str>, +) -> Result> { + let live: Vec<&ManagedServiceRecord> = records.iter().filter(|r| r.is_live()).collect(); + match service_id { + Some(service_id) => { + if let Some(record) = live.iter().find(|r| r.service_id == service_id) { + return Ok(vec![record]); + } + match records.iter().find(|r| r.service_id == service_id) { + // Stopped servers are listed only with `--all`; the restart + // command is the one that list prints for this server. + Some(record) => bail!( + "Server `{service_id}` is not running (status: {}), so it has no API docs. \ + Start it again with `{}`; `rocm services list --all` shows it.", + record.status, + crate::service_hints::service_action_hint("restart", service_id) + ), + // The not-found error every service-id command shares. + None => Err(rocm_core::ManagedServiceNotFound { + service_id: service_id.to_owned(), + } + .into()), + } + } + None if live.is_empty() => bail!( + "No model server is running, so there are no API docs to serve. \ + Start one with `rocm serve `." + ), + None => Ok(live), + } +} + +/// Start the docs server on the requested port, or on `default_port` when +/// none was requested, falling back to a free port when that one is taken. +/// An explicitly requested port is never swapped for another. Returns the +/// server and, after a fallback, the line saying so. +fn start_docs( + paths: &AppPaths, + requested: Option, + default_port: u16, + start: impl Fn(AppPaths, u16) -> Result, +) -> Result<(S, Option)> { + if let Some(port) = requested { + return Ok((start(paths.clone(), port)?, None)); + } + match start(paths.clone(), default_port) { + Ok(server) => Ok((server, None)), + // Only a port that is in use is worked around; any other failure + // would recur on the free port and is reported as it is. + Err(error) if !is_addr_in_use(&error) => Err(error), + Err(error) => { + let server = start(paths.clone(), 0)?; + Ok(( + server, + Some(format!( + "note: port {default_port} is not available ({error:#}); serving on a free port" + )), + )) + } + } +} + +fn is_addr_in_use(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::AddrInUse) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + fn record(paths: &AppPaths, id: &str, status: &str) -> ManagedServiceRecord { + let mut record = ManagedServiceRecord::new( + paths, + id, + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 8000, + "managed", + 0, + None, + None, + None, + ); + record.status = status.to_owned(); + record + } + + #[test] + fn docs_targets_cover_every_state_a_server_can_be_in() { + let (root, paths) = crate::tests::test_paths("services-docs-targets"); + let records = vec![ + record(&paths, "svc-live", "ready"), + record(&paths, "svc-starting", "starting"), + record(&paths, "svc-stopped", "stopped"), + ]; + let ids = |targets: Vec<&ManagedServiceRecord>| -> Vec { + targets.iter().map(|r| r.service_id.clone()).collect() + }; + + assert_eq!( + ids(docs_targets(&records, None).unwrap()), + ["svc-live", "svc-starting"], + "every running server, and only those" + ); + assert_eq!( + ids(docs_targets(&records, Some("svc-live")).unwrap()), + ["svc-live"] + ); + let stopped = docs_targets(&records, Some("svc-stopped")).unwrap_err(); + assert_eq!( + stopped.to_string(), + "Server `svc-stopped` is not running (status: stopped), so it has no API docs. \ + Start it again with `rocm services restart svc-stopped --yes`; \ + `rocm services list --all` shows it." + ); + let unknown = docs_targets(&records, Some("nope")).unwrap_err(); + assert_eq!( + unknown.to_string(), + "No local server named `nope`. See `rocm services list --all`." + ); + let none_running = docs_targets(&records[2..], None).unwrap_err(); + assert!( + none_running + .to_string() + .starts_with("No model server is running"), + "{none_running}" + ); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn a_taken_default_port_falls_back_to_a_free_one_but_a_requested_port_does_not() { + let (root, paths) = crate::tests::test_paths("services-docs-ports"); + let tried = RefCell::new(Vec::new()); + let taken_default = |_: AppPaths, port: u16| -> Result { + tried.borrow_mut().push(port); + if port == 11_430 { + return Err(anyhow::Error::from(std::io::Error::from( + std::io::ErrorKind::AddrInUse, + )) + .context("failed to bind API docs server to 127.0.0.1:11430")); + } + Ok(port) + }; + + let (bound, note) = start_docs(&paths, None, 11_430, taken_default).unwrap(); + assert_eq!(bound, 0, "fell back to a free port"); + assert_eq!(*tried.borrow(), [11_430, 0]); + assert_eq!( + note.as_deref(), + Some( + "note: port 11430 is not available (failed to bind API docs server to \ + 127.0.0.1:11430: address in use); serving on a free port" + ) + ); + + tried.borrow_mut().clear(); + let error = start_docs(&paths, Some(11_430), 11_430, taken_default).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to bind API docs server to 127.0.0.1:11430" + ); + assert_eq!( + *tried.borrow(), + [11_430], + "a requested port is never swapped" + ); + + tried.borrow_mut().clear(); + let (bound, note) = start_docs(&paths, None, 11_431, taken_default).unwrap(); + assert_eq!( + (bound, note), + (11_431, None), + "a free default port is used as is" + ); + + // A failure that is not the port being taken would recur on any + // port, so it is reported rather than retried elsewhere. + tried.borrow_mut().clear(); + let broken = |_: AppPaths, port: u16| -> Result { + tried.borrow_mut().push(port); + bail!("failed to build the API docs runtime") + }; + let error = start_docs(&paths, None, 11_430, broken).unwrap_err(); + assert_eq!(error.to_string(), "failed to build the API docs runtime"); + assert_eq!(*tried.borrow(), [11_430], "no fallback for other errors"); + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn a_launched_server_gets_engine_docs_only_when_its_engine_serves_them() { + let (root, paths) = crate::tests::test_paths("services-docs-launch-rows"); + record(&paths, "svc-vllm", "ready").write().unwrap(); + let mut lemonade = record(&paths, "svc-lemonade", "ready"); + lemonade.engine = "lemonade".to_owned(); + lemonade.write().unwrap(); + + assert_eq!( + launch_docs_rows(&paths, "svc-vllm"), + [ + ("docs", "rocm services docs svc-vllm".to_owned()), + ("engine docs", "http://127.0.0.1:8000/docs".to_owned()), + ] + ); + assert_eq!( + launch_docs_rows(&paths, "svc-lemonade"), + [("docs", "rocm services docs svc-lemonade".to_owned())] + ); + assert_eq!( + launch_docs_rows(&paths, "svc-missing"), + [("docs", "rocm services docs svc-missing".to_owned())], + "an unreadable record still names the command, which reports it" + ); + let _ = std::fs::remove_dir_all(root); + } + + /// `rocmd` cannot call the engines; `rocm` links them, so the check that + /// the docs describe the routes each engine reports lives here. Each + /// engine is asked for its `Endpoint` answer the way the CLI asks it, + /// against a state file like the one its launch writes. + #[test] + fn documented_routes_are_the_routes_each_engine_reports() { + let (root, paths) = crate::tests::test_paths("services-docs-engine-routes"); + paths.ensure().unwrap(); + let service_id = "svc-routes"; + let mut reported = Vec::new(); + for engine in crate::cli::SUPPORTED_ENGINES { + let state_dir = paths.engine_state_dir(engine); + std::fs::create_dir_all(&state_dir).unwrap(); + std::fs::write( + state_dir.join(format!("{service_id}.json")), + r#"{"status":"running","host":"127.0.0.1","port":9,"endpoint_url":"http://127.0.0.1:9/v1"}"#, + ) + .unwrap(); + let endpoint: rocm_engine_protocol::EndpointResponse = + crate::engine_ipc::engine_request( + Some(&paths), + engine, + rocm_engine_protocol::EngineMethod::Endpoint, + &rocm_engine_protocol::EndpointRequest { + service_id: service_id.to_owned(), + }, + ) + .unwrap_or_else(|error| panic!("{engine}: {error:#}")); + reported.push((engine, endpoint.supported_routes)); + } + let _ = std::fs::remove_dir_all(root); + + for (engine, routes) in &reported { + assert_eq!(&rocmd::documented_route_paths(engine), routes, "{engine}"); + } + // Lemonade's health route differs from the others: an engine + // reporting another engine's routes cannot pass for its own. + assert_ne!(reported[0].1, reported[1].1); + } + + /// A server whose process died, though its record still says "ready", + /// is not running to the docs, as it is not to `rocm services list`. + #[test] + fn a_server_whose_process_died_has_no_docs() { + let (root, paths) = crate::tests::test_paths("services-docs-dead-pid"); + paths.ensure().unwrap(); + std::fs::create_dir_all(paths.services_dir()).unwrap(); + let mut exited = if cfg!(windows) { + std::process::Command::new("cmd") + .args(["/C", "exit 0"]) + .spawn() + .unwrap() + } else { + std::process::Command::new("true").spawn().unwrap() + }; + exited.wait().unwrap(); + // Nothing listens on the recorded port either. + let dead_port = std::net::TcpListener::bind(("127.0.0.1", 0)) + .unwrap() + .local_addr() + .unwrap() + .port(); + let mut crashed = record(&paths, "svc-crashed", "ready"); + crashed.port = dead_port; + crashed.supervisor_pid = exited.id(); + crashed.write().unwrap(); + assert!(crashed.is_live(), "premise: the record still says it runs"); + + let server = rocmd::start_api_docs_server(paths, 0, services_loader()).unwrap(); + // Not proxied: the docs are on loopback. + let agent = ureq::AgentBuilder::new().try_proxy_from_env(false).build(); + let index = agent + .get(&format!("{}/", server.base_url)) + .call() + .unwrap() + .into_string() + .unwrap(); + let (status, page) = match agent.get(&server.service_docs_url("svc-crashed")).call() { + Ok(response) => (response.status(), response.into_string().unwrap()), + Err(ureq::Error::Status(status, response)) => (status, response.into_string().unwrap()), + Err(error) => panic!("docs request failed: {error}"), + }; + server.stop(); + let _ = std::fs::remove_dir_all(root); + + assert!(!index.contains("svc-crashed"), "{index}"); + assert!(index.contains("No model server is running"), "{index}"); + assert_eq!(status, 404); + assert!(page.contains("is not running"), "{page}"); + } + + #[test] + fn the_docs_hint_names_this_command_with_the_server_id() { + // The hint text only; that the named command serves the docs is + // scenario service-docs-01, which runs the line `rocm services list` + // printed. + assert_eq!(docs_hint("svc-1"), "rocm services docs svc-1"); + } +} diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index cd6af59fb6..788b4a30a0 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -463,15 +463,22 @@ pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result Result<()> { + let mut rendered = Vec::new(); + for (name, engine, status) in [ + ("services-docs-vllm", "vllm", "starting"), + ("services-docs-lemonade", "lemonade", "starting"), + ("services-docs-stopped", "vllm", "stopped"), + ] { + let (root, paths) = test_paths(name); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-x", + engine, + "qwen", + "Qwen/Qwen3.5", + "0.0.0.0", + 11436, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = status.to_owned(); + record.write()?; + rendered.push(render_services_text(&paths, true)?); + let _ = fs::remove_dir_all(root); + } + + let [vllm, lemonade, stopped] = &rendered[..] else { + unreachable!() + }; + assert!( + vllm.contains("\n docs: rocm services docs svc-x\n"), + "{vllm}" + ); + assert!( + vllm.contains("\n engine docs: http://127.0.0.1:11436/docs\n"), + "{vllm}" + ); + assert!( + lemonade.contains("\n docs: rocm services docs svc-x\n"), + "{lemonade}" + ); + assert!(!lemonade.contains("engine docs:"), "{lemonade}"); + assert!(stopped.contains("status: stopped"), "{stopped}"); + assert!(!stopped.contains("docs:"), "{stopped}"); + Ok(()) + } + #[test] fn log_tail_yields_only_new_complete_segments() -> Result<()> { let (root, paths) = test_paths("log-tail"); diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 70587d7456..3821f4e041 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -17,6 +17,11 @@ e2e-test-hooks = ["rocm-core/e2e-test-hooks"] anyhow.workspace = true async-stream.workspace = true axum.workspace = true +# Forwards the API docs' try-it requests to the model server, over plain +# HTTP to the address its record names (loopback for a server bound to every +# interface), so no TLS stack; `stream` relays streamed completions as they +# arrive. +reqwest = { version = "0.12", default-features = false, features = ["stream"] } clap.workspace = true rocm-core = { path = "../../crates/rocm-core" } rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol" } @@ -25,6 +30,11 @@ serde_json.workspace = true sha2 = "0.10" tokio.workspace = true ureq = { version = "2.12", features = ["native-certs"] } +# API docs for running model servers: OpenAPI model + Swagger UI. `vendored` +# compiles the Swagger UI assets in, so the build downloads nothing and the +# page works offline. +utoipa = "6" +utoipa-swagger-ui = { version = "10", default-features = false, features = ["vendored"] } [dev-dependencies] # The fake engine child both engine IPC callers are tested against. diff --git a/apps/rocmd/src/anthropic_proxy.rs b/apps/rocmd/src/anthropic_proxy.rs index 76486a9153..90f769846c 100644 --- a/apps/rocmd/src/anthropic_proxy.rs +++ b/apps/rocmd/src/anthropic_proxy.rs @@ -82,12 +82,7 @@ impl AnthropicProxyConfig { /// Where to reach the engine from this machine, even when it listens on a /// wildcard address. fn upstream_url(&self) -> String { - let host = match self.host.as_str() { - "0.0.0.0" => "127.0.0.1", - "::" => "::1", - other => other, - }; - format_http_base_url(host, self.engine_port) + format_http_base_url(rocm_core::reachable_host(&self.host), self.engine_port) } } @@ -1656,6 +1651,10 @@ mod tests { }; assert_eq!(config("0.0.0.0").upstream_url(), "http://127.0.0.1:8000"); assert_eq!(config("::").upstream_url(), "http://[::1]:8000"); + assert_eq!( + config("0:0:0:0:0:0:0:0").upstream_url(), + "http://[::1]:8000" + ); assert_eq!(config("127.0.0.1").upstream_url(), "http://127.0.0.1:8000"); } diff --git a/apps/rocmd/src/api_docs.rs b/apps/rocmd/src/api_docs.rs new file mode 100644 index 0000000000..1666cdd013 --- /dev/null +++ b/apps/rocmd/src/api_docs.rs @@ -0,0 +1,1955 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! API docs for every running model server, served on loopback for as long as +//! `rocm services docs` runs in the foreground. +//! +//! Of the servers rocm-cli runs, only vLLM generates its own docs; Lemonade and +//! the llama.cpp server behind it serve none. So this server describes each +//! running server from the routes rocm-cli knows it serves ([`routes_for`]), +//! renders that with Swagger UI, and forwards the page's "Try it out" requests +//! to the server through `/services//api/...`. Forwarding keeps try-it +//! same-origin, so it works on every engine whatever that engine's own +//! cross-origin settings are; Lemonade, for one, only admits browser requests +//! from its own origin. +//! +//! It lives here, beside the daemon, rather than in `rocm`, which runs it +//! in-process: `rocm` depends on `rocmd`, so the HTTP stack it needs stays in +//! one crate. It runs only while `rocm services docs` does; nothing in the +//! background serves it. +//! +//! Which servers are running comes from the [`ServiceLoader`] the caller +//! passes, asked again on each request (at most once per +//! [`SERVICE_LIST_TTL`]). `rocm` passes the loader `rocm services list` uses, +//! which checks each server's process and endpoint, so a server that stops or +//! crashes while the docs are open drops off them. + +use anyhow::{Context, Result, bail}; +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::{DefaultBodyLimit, Path, Request, State}; +use axum::http::{HeaderMap, HeaderValue, Method, StatusCode, Uri, header}; +use axum::middleware::{self, Next}; +use axum::response::{Html, IntoResponse, Response}; +use axum::routing::{any, get}; +use rocm_core::{AppPaths, DEFAULT_LOCAL_HOST, ManagedServiceRecord}; +use serde_json::{Value, json}; +use std::fmt::Write as _; +use std::net::IpAddr; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::net::TcpListener; +use utoipa::openapi::path::{HttpMethod, OperationBuilder, PathItem, PathsBuilder}; +use utoipa::openapi::request_body::RequestBodyBuilder; +use utoipa::openapi::security::{HttpAuthScheme, HttpBuilder, SecurityRequirement, SecurityScheme}; +use utoipa::openapi::{ + ComponentsBuilder, ContentBuilder, InfoBuilder, OpenApi, OpenApiBuilder, Required, + ResponseBuilder, ServerBuilder, +}; + +/// Largest request body the try-it forwarder accepts. Generous enough for a +/// long chat history or a large prompt; the docs server is loopback-only, so +/// this bounds memory, not abuse from elsewhere. +const FORWARD_BODY_LIMIT_BYTES: usize = 64 * 1024 * 1024; + +/// How long the forwarder waits for the model server to accept a connection. +/// There is deliberately no overall timeout: a long completion may stream for +/// minutes. +const FORWARD_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +/// The OpenAPI security scheme name used for servers started with an endpoint +/// key; Swagger UI's "Authorize" box fills it in. +const ENDPOINT_KEY_SCHEME: &str = "endpoint_key"; + +/// How long a stopping docs server lets in-flight requests finish. A try-it +/// answer may stream for minutes and the forwarder sets no overall timeout, so +/// without this bound an open stream would hold up Ctrl+C. +const SHUTDOWN_GRACE: Duration = Duration::from_secs(2); + +/// How long one read of the running servers is reused. Opening a docs page +/// sends several requests at once, and the loader `rocm` passes probes each +/// server's endpoint; requests that arrive while a read is under way wait for +/// it, so a page load costs one read. A server that stops drops off once the +/// read after it stopped completes: at most this long after the last read, +/// plus the time the next read takes. +const SERVICE_LIST_TTL: Duration = Duration::from_secs(1); + +/// `Content-Security-Policy` on every page this server serves itself: no +/// other site may frame it, so a framed docs page cannot be clicked through +/// into sending a try-it request from this origin. +const PAGE_CSP: &str = "frame-ancestors 'none'"; + +/// `Content-Security-Policy` on forwarded answers: as [`PAGE_CSP`], and +/// sandboxed, since the answer comes from the model server but is served from +/// the docs' origin. +const FORWARDED_CSP: &str = "sandbox; frame-ancestors 'none'"; + +/// Reads every service record, refreshed however the caller judges a server +/// to be running; the docs list the ones [`ManagedServiceRecord::is_live`] +/// accepts afterwards. +pub type ServiceLoader = Arc Result> + Send + Sync>; + +/// The port `rocm services docs` serves on unless told otherwise. Kept clear +/// of `DEFAULT_LOCAL_PORT` and the ports above it that consecutive +/// `rocm serve` launches pick. +pub const DEFAULT_API_DOCS_PORT: u16 = 11_430; + +/// A running docs server. It runs on a thread and runtime of its own, so the +/// caller's thread stays free to wait for Ctrl+C. +pub struct ApiDocsServer { + /// `http://127.0.0.1:`, the port actually bound. + pub base_url: String, + shutdown: tokio::sync::oneshot::Sender<()>, + thread: std::thread::JoinHandle<()>, +} + +impl ApiDocsServer { + /// The API docs page of one server. + #[must_use] + pub fn service_docs_url(&self, service_id: &str) -> String { + format!("{}{}/docs/", self.base_url, service_path(service_id)) + } + + /// Stop accepting requests and wait for the server thread to finish. + pub fn stop(self) { + let _ = self.shutdown.send(()); + let _ = self.thread.join(); + } +} + +/// The last read of the running servers, and when it was made. +type CachedServices = Option<(Instant, Arc>)>; + +#[derive(Clone)] +struct DocsState { + paths: Arc, + load: ServiceLoader, + /// Held across a read, so concurrent requests share one. + cached: Arc>, + service_list_ttl: Duration, + client: reqwest::Client, + /// The origins of this server's own pages, the only ones whose browser + /// requests are admitted. + own_origins: Arc<[String]>, +} + +/// Serve API docs for the running model servers on `127.0.0.1:`; port +/// 0 binds a free one, named in [`ApiDocsServer::base_url`]. `load` reads the +/// service records; see [`ServiceLoader`]. +/// +/// # Errors +/// Fails when the port cannot be bound or the server cannot start. +pub fn start_api_docs_server( + paths: AppPaths, + port: u16, + load: ServiceLoader, +) -> Result { + start_with(paths, port, load, FORWARD_CONNECT_TIMEOUT, SERVICE_LIST_TTL) +} + +fn start_with( + paths: AppPaths, + port: u16, + load: ServiceLoader, + connect_timeout: Duration, + service_list_ttl: Duration, +) -> Result { + let listener = std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, port)).with_context(|| { + format!("failed to bind API docs server to {DEFAULT_LOCAL_HOST}:{port}") + })?; + let addr = listener + .local_addr() + .context("failed to read API docs listener address")?; + if !addr.ip().is_loopback() { + bail!("API docs server must bind to loopback; resolved address was {addr}"); + } + listener + .set_nonblocking(true) + .context("failed to configure the API docs listener")?; + let bound = addr.port(); + let app = router(DocsState { + paths: Arc::new(paths), + load, + cached: Arc::new(tokio::sync::Mutex::new(None)), + service_list_ttl, + client: forwarding_client(connect_timeout)?, + own_origins: [ + format!("http://{DEFAULT_LOCAL_HOST}:{bound}"), + format!("http://localhost:{bound}"), + ] + .into(), + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .context("failed to build the API docs runtime")?; + // Registered here rather than on the server thread, so a failure is this + // call's error instead of a thread that ends while the caller waits. + let listener = { + let _entered = runtime.enter(); + TcpListener::from_std(listener).context("failed to register the API docs listener")? + }; + let (shutdown, shutdown_received) = tokio::sync::oneshot::channel::<()>(); + let thread = std::thread::Builder::new() + .name("rocmd-api-docs".to_owned()) + .spawn(move || { + runtime.block_on(async move { + // `axum::serve` retries failed accepts rather than returning + // an error, so this ends only when the caller stops it. + if let Err(error) = serve_until_stopped(listener, app, shutdown_received).await { + eprintln!("API docs server stopped: {error}"); + } + }); + }) + .context("failed to start the API docs thread")?; + Ok(ApiDocsServer { + base_url: format!("http://{addr}"), + shutdown, + thread, + }) +} + +/// Serve until `shutdown` fires, then give in-flight requests +/// [`SHUTDOWN_GRACE`] to finish before dropping them. +async fn serve_until_stopped( + listener: TcpListener, + app: Router, + shutdown: tokio::sync::oneshot::Receiver<()>, +) -> Result<()> { + let stopping = Arc::new(tokio::sync::Notify::new()); + let stop_signal = Arc::clone(&stopping); + let serve = axum::serve(listener, app).with_graceful_shutdown(async move { + let _ = shutdown.await; + stop_signal.notify_one(); + }); + tokio::select! { + served = serve => served.map_err(anyhow::Error::from), + () = async { + stopping.notified().await; + tokio::time::sleep(SHUTDOWN_GRACE).await; + } => Ok(()), + } +} + +/// The client the forwarder sends try-it requests with. +fn forwarding_client(connect_timeout: Duration) -> Result { + reqwest::Client::builder() + .connect_timeout(connect_timeout) + // The model server is on this machine. A proxy configured for the + // user's outbound traffic (`HTTP_PROXY`, `ALL_PROXY`) would carry the + // prompt and the endpoint key off it, or fail the request outright. + .no_proxy() + // A redirect is not followed: the browser gets the model server's own + // status. The routes documented here never redirect. + .redirect(reqwest::redirect::Policy::none()) + .build() + .context("failed to build the API docs forwarding client") +} + +fn router(state: DocsState) -> Router { + Router::new() + .route("/", get(index_page)) + .route("/services/{id}/openapi.json", get(service_openapi_json)) + .route("/services/{id}/docs", get(docs_redirect)) + .route("/services/{id}/docs/", get(docs_index)) + .route("/services/{id}/docs/{*file}", get(docs_file)) + .route( + "/services/{id}/api/{*path}", + any(forward).layer(DefaultBodyLimit::max(FORWARD_BODY_LIMIT_BYTES)), + ) + .layer(middleware::from_fn_with_state( + state.clone(), + require_loopback_host, + )) + .with_state(state) +} + +/// Refuse requests whose `Host` is not a loopback name, and browser requests +/// sent from any page but this server's own. The listener is already +/// loopback-only; the `Host` check closes DNS rebinding, where a web page +/// resolves its own hostname to 127.0.0.1. A browser request is recognised by +/// its `Origin`, which a page's `fetch` and form posts carry and which must +/// name this server, or, when it has none (a link, an image, a top-level +/// navigation), by `Sec-Fetch-Site`, which must say the request came from +/// this server's own pages or from the user typing the address. Requests from +/// other programs (curl) carry neither and are admitted. So no other page the +/// user has open, another local web app included, can send a request through +/// the forwarder. +/// +/// Every answer is marked so no other site can frame it or have it sniffed +/// into a different type. +async fn require_loopback_host( + State(state): State, + request: Request, + next: Next, +) -> Response { + if !request_is_admitted(request.headers(), &state.own_origins) { + return mark_as_not_frameable( + ( + StatusCode::FORBIDDEN, + "API docs answer only their own pages, on 127.0.0.1 or localhost\n", + ) + .into_response(), + ); + } + mark_as_not_frameable(next.run(request).await) +} + +fn request_is_admitted(headers: &HeaderMap, own_origins: &[String]) -> bool { + let text = |name| headers.get(name).map(|value: &HeaderValue| value.to_str()); + let host_allowed = + matches!(text(header::HOST), Some(Ok(host)) if host_header_is_loopback(host)); + let browser_allowed = match (text(header::ORIGIN), text(SEC_FETCH_SITE)) { + (Some(origin), _) => origin.is_ok_and(|origin| origin_is_own(origin, own_origins)), + (None, Some(site)) => site.is_ok_and(fetch_site_is_own), + (None, None) => true, + }; + host_allowed && browser_allowed +} + +/// The fetch-metadata header browsers send with every request, `Origin` or +/// not. +const SEC_FETCH_SITE: header::HeaderName = header::HeaderName::from_static("sec-fetch-site"); + +/// `same-origin` is a request from this server's own pages; `none` is one the +/// user started (typing the address, a bookmark). `same-site` is refused too: +/// another port on 127.0.0.1 is the same site. +fn fetch_site_is_own(site: &str) -> bool { + matches!(site, "same-origin" | "none") +} + +fn mark_as_not_frameable(mut response: Response) -> Response { + let headers = response.headers_mut(); + headers + .entry(header::CONTENT_SECURITY_POLICY) + .or_insert(HeaderValue::from_static(PAGE_CSP)); + headers.insert(header::X_FRAME_OPTIONS, HeaderValue::from_static("DENY")); + headers.insert( + header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ); + response +} + +fn host_header_is_loopback(host: &str) -> bool { + let name = if let Some(rest) = host.strip_prefix('[') { + rest.split(']').next().unwrap_or_default() + } else { + host.rsplit_once(':').map_or(host, |(name, _)| name) + }; + name.eq_ignore_ascii_case("localhost") + || name.parse::().is_ok_and(|ip| ip.is_loopback()) +} + +/// Whether an `Origin` header names one of this server's own pages. Browsers +/// send the scheme and host lowercased and the port only when it is not the +/// scheme's default, so an exact comparison is enough. +fn origin_is_own(origin: &str, own_origins: &[String]) -> bool { + own_origins.iter().any(|own| own == origin) +} + +/// Every running server, as the [`ServiceLoader`] judges it. A failure to +/// read the records is logged and returned, so a page can say the records +/// could not be read rather than claim that nothing is running. The loader +/// may probe each server, so it runs off the async thread. Requests that +/// arrive during a read wait for it rather than starting their own, and reuse +/// its result when it succeeds; a failed read is not remembered, so each +/// waiter then reads again in turn. The loader must therefore bound its own +/// probes, as the one `rocm` passes does: each server's liveness check and +/// inference probe have their own timeouts (750 ms and +/// `rocm_core::INFERENCE_PROBE_TIMEOUT`, 8 s), so a read takes at most about +/// nine seconds per server, servers probed in turn. A timeout here would only +/// abandon the read, not stop it. +async fn running_services(state: &DocsState) -> Result>> { + let mut cached = state.cached.lock().await; + if let Some((read_at, services)) = cached.as_ref() + && read_at.elapsed() < state.service_list_ttl + { + return Ok(Arc::clone(services)); + } + let load = Arc::clone(&state.load); + let paths = Arc::clone(&state.paths); + let loaded = tokio::task::spawn_blocking(move || load(&paths)) + .await + .context("the service records read stopped") + .and_then(|loaded| loaded) + .inspect_err(|error| { + eprintln!("API docs could not read the service records: {error:#}"); + })?; + let services: Arc> = Arc::new( + loaded + .into_iter() + .filter(ManagedServiceRecord::is_live) + .collect(), + ); + *cached = Some((Instant::now(), Arc::clone(&services))); + Ok(services) +} + +/// The running server with this id. Looked up by matching the id against the +/// records rather than by building a path from it, so a crafted id cannot +/// reach outside the services folder. +async fn find_running_service( + state: &DocsState, + service_id: &str, +) -> Result> { + Ok(running_services(state) + .await? + .iter() + .find(|record| record.service_id == service_id) + .cloned()) +} + +/// `/services/`, with the id percent-encoded: an id may hold characters +/// (a space, `#`, `?`, `%`) that would otherwise end or change the path. +fn service_path(service_id: &str) -> String { + let mut path = String::from("/services/"); + for byte in service_id.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + path.push(char::from(byte)); + } else { + let _ = write!(path, "%{byte:02X}"); + } + } + path +} + +/// The answer for a server lookup that could not read the records: 503, since +/// the server may well be running. +fn records_unreadable(error: &anyhow::Error) -> Response { + ( + StatusCode::SERVICE_UNAVAILABLE, + axum::Json(json!({ + "error": format!("could not read the service records: {error:#}"), + })), + ) + .into_response() +} + +fn not_running_page(service_id: &str) -> Response { + let id = html_escape(service_id); + ( + StatusCode::NOT_FOUND, + Html(format!( + "Server not running\ +

Server {id} is not running

\ +

Its API docs are available while it runs. \ + Running servers

" + )), + ) + .into_response() +} + +async fn index_page(State(state): State) -> Response { + let services = match running_services(&state).await { + Ok(services) => services, + Err(error) => return records_unreadable(&error), + }; + let mut rows = String::new(); + for record in services.iter() { + let _ = write!( + rows, + "
  • {model} ({engine}, {id}, {endpoint})
  • ", + // Percent-encoded, so it holds no character HTML treats specially. + href = service_path(&record.service_id), + id = html_escape(&record.service_id), + model = html_escape(record.served_model_name()), + engine = html_escape(&record.engine), + endpoint = html_escape(&record.endpoint_url), + ); + } + let body = if services.is_empty() { + "

    No model server is running. Start one with rocm serve.

    ".to_owned() + } else { + format!("
      {rows}
    ") + }; + Html(format!( + "ROCm CLI API docs\ +

    Running model servers

    {body}" + )) + .into_response() +} + +async fn service_openapi_json( + State(state): State, + Path(service_id): Path, +) -> Response { + match find_running_service(&state, &service_id).await { + Ok(Some(record)) => axum::Json(service_openapi(&record)).into_response(), + Err(error) => records_unreadable(&error), + Ok(None) => ( + StatusCode::NOT_FOUND, + axum::Json(json!({ + "error": format!("server {service_id} is not running"), + })), + ) + .into_response(), + } +} + +/// `/services//docs` to the page itself, whose assets load relative to +/// the trailing slash. Only a running server's id is redirected, and +/// percent-encoded again, since it comes decoded from the request. The +/// redirect is temporary, since the server it names can stop. +async fn docs_redirect(State(state): State, Path(service_id): Path) -> Response { + match find_running_service(&state, &service_id).await { + Ok(Some(record)) => { + match HeaderValue::try_from(format!("{}/docs/", service_path(&record.service_id))) { + Ok(location) => ( + StatusCode::TEMPORARY_REDIRECT, + [(header::LOCATION, location)], + ) + .into_response(), + Err(_) => not_running_page(&service_id), + } + } + Ok(None) => not_running_page(&service_id), + Err(error) => records_unreadable(&error), + } +} + +async fn docs_index(State(state): State, Path(service_id): Path) -> Response { + serve_swagger_file(&state, &service_id, "").await +} + +async fn docs_file( + State(state): State, + Path((service_id, file)): Path<(String, String)>, +) -> Response { + serve_swagger_file(&state, &service_id, &file).await +} + +async fn serve_swagger_file(state: &DocsState, service_id: &str, file: &str) -> Response { + match find_running_service(state, service_id).await { + Ok(Some(_)) => {} + Ok(None) => return not_running_page(service_id), + Err(error) => return records_unreadable(&error), + } + let config = Arc::new( + utoipa_swagger_ui::Config::new([format!("{}/openapi.json", service_path(service_id))]) + .try_it_out_enabled(true) + // The endpoint key typed into "Authorize" lasts until the page is + // closed; it is not saved in the browser, where it would outlive + // the server it belongs to. + .persist_authorization(false), + ); + match utoipa_swagger_ui::serve(file, config) { + Ok(Some(swagger_file)) => ( + [(header::CONTENT_TYPE, swagger_file.content_type)], + swagger_file.bytes.into_owned(), + ) + .into_response(), + Ok(None) => StatusCode::NOT_FOUND.into_response(), + Err(error) => (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(), + } +} + +/// Relay one try-it request to the model server and stream its answer back. +async fn forward( + State(state): State, + Path((service_id, _decoded_path)): Path<(String, String)>, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let record = match find_running_service(&state, &service_id).await { + Ok(Some(record)) => record, + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + axum::Json(json!({ "error": format!("server {service_id} is not running") })), + ) + .into_response(); + } + Err(error) => return records_unreadable(&error), + }; + // The path exactly as the browser sent it. The extracted one is + // percent-decoded: an encoded `?` or `/` in it would turn into a query or + // an extra segment, and the request would reach a different route. + let Some(path) = raw_forwarded_path(uri.path()) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let mut target = format!("{}/{path}", record.local_base_url()); + if let Some(query) = uri.query() { + target.push('?'); + target.push_str(query); + } + let mut request = state.client.request(method, &target).body(body); + // Only what the routes need: the body's type, the answer the caller + // accepts, and the endpoint key the user typed into "Authorize". Nothing + // the browser adds on its own (cookies, origin) reaches the model server. + for name in [header::CONTENT_TYPE, header::ACCEPT, header::AUTHORIZATION] { + if let Some(value) = headers.get(&name) { + request = request.header(name, value); + } + } + match request.send().await { + Ok(answer) => { + let status = answer.status(); + let content_type = answer.headers().get(header::CONTENT_TYPE).cloned(); + let mut response = Response::new(Body::from_stream(answer.bytes_stream())); + *response.status_mut() = status; + let headers = response.headers_mut(); + if let Some(content_type) = content_type { + headers.insert(header::CONTENT_TYPE, content_type); + } + // The answer comes from the model server but is served from the + // docs' origin. A page in it (llama.cpp's own UI, say) must not + // run there: it is shown as data, never as a document of this + // origin. `nosniff` is added for every answer, by + // `require_loopback_host`. + headers.insert( + header::CONTENT_SECURITY_POLICY, + HeaderValue::from_static(FORWARDED_CSP), + ); + response + } + Err(error) => ( + StatusCode::BAD_GATEWAY, + [( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + )], + json!({ + "error": format!( + "could not reach server {service_id} at {}: {error}", + record.local_base_url() + ), + }) + .to_string(), + ) + .into_response(), + } +} + +/// The part of `/services//api/` after `api/`, still percent-encoded. +fn raw_forwarded_path(request_path: &str) -> Option<&str> { + let mut segments = request_path.splitn(5, '/'); + match ( + segments.next(), + segments.next(), + segments.next(), + segments.next(), + segments.next(), + ) { + (Some(""), Some("services"), Some(_), Some("api"), Some(path)) => Some(path), + _ => None, + } +} + +/// One route a model server answers, as the docs describe it. +struct RouteDoc { + method: HttpMethod, + path: &'static str, + summary: &'static str, + description: &'static str, + /// An example request body, built for the server's real model id. + example_body: Option Value>, +} + +/// The paths the docs describe for a server of `engine`, in order. `rocm`'s +/// tests compare them with what each engine reports as `supported_routes`. +#[must_use] +pub fn documented_route_paths(engine: &str) -> Vec { + routes_for(engine) + .iter() + .map(|route| route.path.to_owned()) + .collect() +} + +/// The routes each engine's server answers: the engine's health route from +/// [`rocm_engine_protocol::ENGINE_HEALTH_ROUTES`], when it has one there, plus +/// the chat routes. An engine the table does not know gets no health route +/// rather than another engine's. +fn routes_for(engine: &str) -> Vec { + let health = rocm_engine_protocol::engine_health_route(engine).map(|path| RouteDoc { + method: HttpMethod::Get, + path, + summary: "Check the server is up", + description: "Answers 200 once the server accepts requests.", + example_body: None, + }); + health.into_iter().chain(chat_routes()).collect() +} + +fn chat_routes() -> [RouteDoc; 3] { + [ + RouteDoc { + method: HttpMethod::Get, + path: "/v1/models", + summary: "List the served models", + description: "OpenAI-compatible model list. The `id` of each entry is the \ + `model` value requests use.", + example_body: None, + }, + RouteDoc { + method: HttpMethod::Post, + path: "/v1/chat/completions", + summary: "Chat completion", + description: "OpenAI-compatible chat completion. Set `stream` to `true` to \ + receive server-sent events as tokens are generated.", + example_body: Some(chat_completion_example), + }, + RouteDoc { + method: HttpMethod::Post, + path: "/v1/completions", + summary: "Text completion", + description: "OpenAI-compatible text completion of a raw prompt.", + example_body: Some(completion_example), + }, + ] +} + +fn chat_completion_example(model: &str) -> Value { + json!({ + "model": model, + "messages": [{ "role": "user", "content": "Say hello in one sentence." }], + "max_tokens": 64, + "stream": false, + }) +} + +fn completion_example(model: &str) -> Value { + json!({ + "model": model, + "prompt": "The capital of France is", + "max_tokens": 16, + }) +} + +/// Where each engine documents more than rocm-cli describes. +fn engine_docs_note(record: &ManagedServiceRecord) -> String { + if let Some(own_docs) = record.engine_docs_url() { + return format!( + "vLLM also documents every route it serves, beyond the ones listed here, at \ + [{own_docs}]({own_docs})." + ); + } + match record.engine.as_str() { + "lemonade" => "On Linux, Lemonade runs chat models with the llama.cpp server, which also \ + answers its own routes; see the \ + [llama.cpp server documentation](https://github.com/ggml-org/llama.cpp/tree/master/tools/server)." + .to_owned(), + "aim" => "This server runs in an AMD Inference Microservice (AIM) container; the routes \ + listed here are the OpenAI-compatible ones rocm-cli uses." + .to_owned(), + _ => "The routes listed here are the OpenAI-compatible ones rocm-cli uses.".to_owned(), + } +} + +fn service_openapi(record: &ManagedServiceRecord) -> OpenApi { + // The name the server answers to, which for AIM is not the image + // reference recorded as the canonical id. + let model = record.served_model_name(); + let mut paths = PathsBuilder::new(); + for route in routes_for(&record.engine) { + let mut operation = OperationBuilder::new() + .summary(Some(route.summary)) + .description(Some(route.description)) + .response( + "200", + ResponseBuilder::new() + .description("The server's answer") + .content("application/json", ContentBuilder::new().build()), + ); + if let Some(example_body) = route.example_body { + operation = operation.request_body(Some( + RequestBodyBuilder::new() + .required(Some(Required::True)) + .content( + "application/json", + ContentBuilder::new() + .example(Some(example_body(model))) + .build(), + ) + .build(), + )); + } + paths = paths.path(route.path, PathItem::new(route.method, operation)); + } + + let description = format!( + "`{model}` served by {engine} at `{endpoint}`.\n\n\ + \"Try it out\" sends requests through this page to the server.\n\n{note}", + engine = record.engine, + endpoint = record.endpoint_url, + note = engine_docs_note(record), + ); + let mut builder = OpenApiBuilder::new() + .info( + InfoBuilder::new() + .title(format!("{model} ({})", record.service_id)) + .version(env!("CARGO_PKG_VERSION")) + .description(Some(description)), + ) + .servers(Some([ServerBuilder::new() + .url(format!("{}/api", service_path(&record.service_id))) + .description(Some(format!( + "{} through the docs page", + record.endpoint_url + ))) + .build()])) + .paths(paths); + if record.requires_api_key { + builder = builder + .components(Some( + ComponentsBuilder::new() + .security_scheme( + ENDPOINT_KEY_SCHEME, + SecurityScheme::Http( + HttpBuilder::new().scheme(HttpAuthScheme::Bearer).build(), + ), + ) + .build(), + )) + .security(Some([SecurityRequirement::new( + ENDPOINT_KEY_SCHEME, + Vec::::new(), + )])); + } + builder.build() +} + +fn html_escape(text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for character in text.chars() { + match character { + '&' => escaped.push_str("&"), + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '"' => escaped.push_str("""), + '\'' => escaped.push_str("'"), + other => escaped.push(other), + } + } + escaped +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::temp_app_paths; + use axum::Json; + use std::fs; + + /// Serializes the tests that set proxy variables. Every other test here + /// talks to its servers through [`test_client`], which ignores them. + static PROXY_ENV_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + /// The test's own HTTP client: never proxied, so a proxy variable set by + /// the proxy test cannot reroute it, and never following a redirect, so a + /// test sees exactly what the docs server answered. + fn test_client() -> reqwest::Client { + reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap() + } + + /// The records exactly as written, the loader the daemon's own code uses. + fn records_as_written() -> ServiceLoader { + Arc::new(|paths: &AppPaths| crate::persistence::load_managed_services(paths)) + } + + fn start(paths: &AppPaths) -> Result { + start_api_docs_server(paths.clone(), 0, records_as_written()) + } + + fn record(paths: &AppPaths, service_id: &str, engine: &str, port: u16) -> ManagedServiceRecord { + let mut record = ManagedServiceRecord::new( + paths, + service_id, + engine, + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record + } + + fn services_dir_ready(paths: &AppPaths) -> Result<()> { + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + Ok(()) + } + + fn operation_paths(openapi: &Value) -> Vec { + openapi["paths"] + .as_object() + .expect("paths object") + .keys() + .cloned() + .collect() + } + + #[test] + fn chat_server_docs_list_its_routes_with_the_served_model() { + let (root, paths) = temp_app_paths("api-docs-chat-routes"); + for (engine, health) in [ + ("vllm", "/health"), + ("lemonade", "/v1/health"), + ("aim", "/health"), + ] { + let mut served = record(&paths, "svc-1", engine, 9); + // As for an AIM server, whose canonical id is the image reference + // and whose served name is adopted from the engine. + served.canonical_model_id = "registry.example/aim/qwen:1.0".to_owned(); + served.served_model_id = Some("Qwen/Qwen3.5".to_owned()); + let openapi = serde_json::to_value(service_openapi(&served)).unwrap(); + assert!( + openapi["info"]["title"] + .as_str() + .is_some_and(|title| title.starts_with("Qwen/Qwen3.5 ")), + "{engine}: {}", + openapi["info"]["title"] + ); + let mut listed = operation_paths(&openapi); + listed.sort(); + let mut expected = vec![ + health.to_owned(), + "/v1/chat/completions".to_owned(), + "/v1/completions".to_owned(), + "/v1/models".to_owned(), + ]; + expected.sort(); + assert_eq!(listed, expected, "{engine}"); + assert_eq!( + openapi["paths"]["/v1/chat/completions"]["post"]["requestBody"]["content"]["application/json"] + ["example"]["model"], + "Qwen/Qwen3.5", + "{engine}: try-it must start from the model the server answers to" + ); + assert_eq!( + openapi["servers"][0]["url"], "/services/svc-1/api", + "{engine}" + ); + assert!( + openapi.get("security").is_none(), + "{engine}: loopback needs no key" + ); + } + fs::remove_dir_all(root).ok(); + } + + /// The docs' own route list against the protocol's table. That each + /// engine reports that table is `rocm`'s + /// `documented_routes_are_the_routes_each_engine_reports`, which can call + /// the engines. + #[test] + fn documented_routes_are_the_protocols_route_table() { + for engine in ["vllm", "lemonade", "aim", "unknown"] { + assert_eq!( + documented_route_paths(engine), + rocm_engine_protocol::engine_supported_routes(engine), + "{engine}" + ); + } + } + + #[test] + fn forwarded_paths_keep_their_encoding() { + assert_eq!( + raw_forwarded_path("/services/svc-1/api/v1/models/Qwen%2FQwen3.5%3Fx"), + Some("v1/models/Qwen%2FQwen3.5%3Fx") + ); + assert_eq!(raw_forwarded_path("/services/svc-1/api/"), Some("")); + assert_eq!(raw_forwarded_path("/services/svc-1/docs/x"), None); + } + + #[test] + fn service_paths_percent_encode_the_id() { + assert_eq!(service_path("svc-1.a_b~"), "/services/svc-1.a_b~"); + assert_eq!(service_path("svc a#1%?/"), "/services/svc%20a%231%25%3F%2F"); + assert_eq!(service_path("é"), "/services/%C3%A9"); + } + + #[test] + fn only_the_docs_servers_own_pages_may_send_browser_requests() { + let own: Vec = vec![ + "http://127.0.0.1:11430".to_owned(), + "http://localhost:11430".to_owned(), + ]; + for origin in ["http://127.0.0.1:11430", "http://localhost:11430"] { + assert!(origin_is_own(origin, &own), "{origin}"); + } + for origin in [ + // Another local web app, on this machine but not these docs. + "http://localhost:3000", + "http://127.0.0.1:11431", + "https://127.0.0.1:11430", + "https://evil.example", + "http://127.0.0.1.evil.example", + "http://127.0.0.1:11430/", + "null", + "", + ] { + assert!(!origin_is_own(origin, &own), "{origin}"); + } + for site in ["same-origin", "none"] { + assert!(fetch_site_is_own(site), "{site}"); + } + // `same-site` covers every port of 127.0.0.1, other local apps too. + for site in ["same-site", "cross-site", "", "SAME-ORIGIN"] { + assert!(!fetch_site_is_own(site), "{site}"); + } + } + + #[test] + fn the_bundled_swagger_ui_is_the_version_named_in_the_notices() { + let config = Arc::new(utoipa_swagger_ui::Config::new(["/openapi.json"])); + let bundle = utoipa_swagger_ui::serve("swagger-ui-bundle.js", config) + .unwrap() + .expect("the vendored Swagger UI ships its bundle"); + let bundle = String::from_utf8_lossy(&bundle.bytes); + let marker = "PACKAGE_VERSION:\""; + let start = bundle.find(marker).expect("the bundle states its version") + marker.len(); + let version: String = bundle[start..].chars().take_while(|c| *c != '"').collect(); + // Read, not `include_str!`: the file is not ASCII, and xtask's output + // contract rejects non-ASCII data compiled into a binary. + let notices = fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../THIRD_PARTY_NOTICES.txt" + )) + .expect("THIRD_PARTY_NOTICES.txt at the repo root"); + assert!( + notices.contains(&format!(" - Swagger UI {version} ")), + "THIRD_PARTY_NOTICES.txt must name the Swagger UI {version} that rocmd serves; \ + update the bundled-assets section of about.hbs and run `cargo xtask tpn`" + ); + } + + #[test] + fn a_taken_port_is_reported_and_port_zero_binds_a_free_one() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-ports"); + paths.ensure()?; + let taken = std::net::TcpListener::bind((DEFAULT_LOCAL_HOST, 0))?; + let taken_port = taken.local_addr()?.port(); + let error = start_api_docs_server(paths.clone(), taken_port, records_as_written()) + .err() + .expect("a taken port cannot be served on"); + assert!( + format!("{error:#}").contains(&format!("127.0.0.1:{taken_port}")), + "{error:#}" + ); + let server = start(&paths)?; + assert!(!server.base_url.ends_with(":0"), "{}", server.base_url); + assert_eq!( + server.service_docs_url("svc-1"), + format!("{}/services/svc-1/docs/", server.base_url) + ); + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[test] + fn keyed_server_docs_ask_for_the_endpoint_key() { + let (root, paths) = temp_app_paths("api-docs-keyed"); + let mut keyed = record(&paths, "svc-1", "vllm", 9); + keyed.requires_api_key = true; + let openapi = serde_json::to_value(service_openapi(&keyed)).unwrap(); + assert_eq!( + openapi["components"]["securitySchemes"][ENDPOINT_KEY_SCHEME]["scheme"], + "bearer" + ); + assert!(openapi["security"][0].get(ENDPOINT_KEY_SCHEME).is_some()); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn engine_note_names_each_engines_own_docs() { + let (root, paths) = temp_app_paths("api-docs-note"); + let note = |engine| engine_docs_note(&record(&paths, "a", engine, 8000)); + assert!(note("vllm").contains("http://127.0.0.1:8000/docs")); + assert!(note("lemonade").contains("llama.cpp")); + for engine in ["aim", "unknown"] { + let text = note(engine); + assert!( + !text.contains("llama.cpp") && !text.contains("/docs"), + "{engine} is not Lemonade or vLLM: {text}" + ); + } + assert!(note("aim").contains("AIM")); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn only_loopback_host_names_are_accepted() { + for host in [ + "127.0.0.1:11430", + "localhost:11430", + "LOCALHOST", + "[::1]:11430", + "127.0.0.1", + ] { + assert!(host_header_is_loopback(host), "{host}"); + } + for host in [ + "evil.example:11430", + "192.168.1.5:11430", + "[2001:db8::1]:11430", + "", + ] { + assert!(!host_header_is_loopback(host), "{host}"); + } + } + + #[test] + fn html_escape_neutralises_markup() { + assert_eq!( + html_escape("'&'"), + "<a href="x">'&'</a>" + ); + } + + fn header_text(headers: &HeaderMap, name: header::HeaderName) -> Option { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned) + } + + /// What reached the stand-in model server, as it saw it: the path still + /// percent-encoded, as the request line carried it. + async fn echo(method: Method, uri: Uri, headers: HeaderMap, body: Bytes) -> Json { + Json(json!({ + "method": method.as_str(), + "path": uri.path(), + "query": uri.query(), + "content_type": header_text(&headers, header::CONTENT_TYPE), + "accept": header_text(&headers, header::ACCEPT), + "authorization": header_text(&headers, header::AUTHORIZATION), + "cookie": header_text(&headers, header::COOKIE), + "body_len": body.len(), + "body": String::from_utf8_lossy(&body), + })) + } + + /// A stand-in model server on `host` that echoes what reached it, so a + /// test can see exactly what the forwarder passed on. `/redirect` + /// answers with a redirect. `None` when `host` cannot be bound here. + async fn start_echo_model_server_on(host: IpAddr) -> Option { + let listener = TcpListener::bind((host, 0)).await.ok()?; + let port = listener.local_addr().ok()?.port(); + let app = Router::new() + .route( + "/redirect", + any(|| async { + ( + StatusCode::TEMPORARY_REDIRECT, + [(header::LOCATION, "/v1/models")], + ) + }), + ) + .fallback(echo) + .layer(DefaultBodyLimit::disable()); + tokio::spawn(async move { axum::serve(listener, app).await.ok() }); + Some(port) + } + + async fn start_echo_model_server() -> u16 { + start_echo_model_server_on(IpAddr::from([127, 0, 0, 1])) + .await + .expect("bind 127.0.0.1") + } + + #[tokio::test(flavor = "multi_thread")] + async fn docs_server_describes_and_forwards_to_a_running_server() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-live"); + services_dir_ready(&paths)?; + let model_port = start_echo_model_server().await; + record(&paths, "svc-live", "vllm", model_port).write()?; + let mut stopped = record(&paths, "svc-stopped", "vllm", model_port); + stopped.status = "stopped".to_owned(); + stopped.write()?; + + let server = start(&paths)?; + let base = server.base_url.clone(); + let client = test_client(); + + let index = client.get(format!("{base}/")).send().await?.text().await?; + assert!(index.contains("/services/svc-live/docs/"), "{index}"); + assert!(!index.contains("svc-stopped"), "{index}"); + + let page = client + .get(format!("{base}/services/svc-live/docs/")) + .send() + .await?; + assert_eq!(page.status(), StatusCode::OK); + assert!(page.text().await?.contains("swagger-ui")); + let initializer = client + .get(format!( + "{base}/services/svc-live/docs/swagger-initializer.js" + )) + .send() + .await? + .text() + .await?; + assert!( + initializer.contains("/services/svc-live/openapi.json"), + "Swagger UI must load this server's description: {initializer}" + ); + assert!( + initializer.contains("\"persistAuthorization\": false"), + "the endpoint key must not be saved in the browser: {initializer}" + ); + + let openapi: Value = client + .get(format!("{base}/services/svc-live/openapi.json")) + .send() + .await? + .text() + .await? + .parse()?; + assert_eq!(openapi["servers"][0]["url"], "/services/svc-live/api"); + + let forwarded: Value = client + .post(format!( + "{base}/services/svc-live/api/v1/chat/completions?trace=1" + )) + .header(header::AUTHORIZATION, "Bearer typed-in-docs") + .header(header::COOKIE, "session=browser-only") + .header(header::CONTENT_TYPE, "application/json") + .header(header::ACCEPT, "text/event-stream") + .body(r#"{"model":"Qwen/Qwen3.5"}"#) + .send() + .await? + .text() + .await? + .parse()?; + assert_eq!(forwarded["method"], "POST"); + assert_eq!(forwarded["path"], "/v1/chat/completions"); + assert_eq!(forwarded["body"], r#"{"model":"Qwen/Qwen3.5"}"#); + assert_eq!(forwarded["query"], "trace=1"); + assert_eq!(forwarded["authorization"], "Bearer typed-in-docs"); + assert_eq!( + forwarded["content_type"], "application/json", + "the server parses the body by it" + ); + assert_eq!( + forwarded["accept"], "text/event-stream", + "a streamed answer is asked for by it" + ); + assert_eq!( + forwarded["cookie"], + Value::Null, + "browser cookies stay in the browser" + ); + + for path in ["docs/", "openapi.json", "api/v1/models"] { + let response = client + .get(format!("{base}/services/svc-stopped/{path}")) + .send() + .await?; + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{path}"); + } + let stopped_page = client + .get(format!("{base}/services/svc-stopped/docs/")) + .send() + .await? + .text() + .await?; + assert!(stopped_page.contains("is not running"), "{stopped_page}"); + + let rebound = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::HOST, "attacker.example") + .body("{}") + .send() + .await?; + assert_eq!(rebound.status(), StatusCode::FORBIDDEN, "DNS rebinding"); + let cross_site = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::ORIGIN, "https://some-site.example") + .header(header::CONTENT_TYPE, "text/plain") + .body("{}") + .send() + .await?; + assert_eq!( + cross_site.status(), + StatusCode::FORBIDDEN, + "cross-site post" + ); + // Another web app on this machine is still not these docs. + let other_local_app = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::ORIGIN, "http://localhost:3000") + .header(header::CONTENT_TYPE, "text/plain") + .body("{}") + .send() + .await?; + assert_eq!( + other_local_app.status(), + StatusCode::FORBIDDEN, + "a page from another local port" + ); + let localhost_alias = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header( + header::ORIGIN, + base.replace("127.0.0.1", "localhost").as_str(), + ) + .body("{}") + .send() + .await?; + assert_eq!( + localhost_alias.status(), + StatusCode::OK, + "the docs page opened as localhost" + ); + let same_origin = client + .post(format!("{base}/services/svc-live/api/v1/chat/completions")) + .header(header::ORIGIN, base.as_str()) + .body("{}") + .send() + .await?; + assert_eq!(same_origin.status(), StatusCode::OK, "the docs page itself"); + + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + /// A browser sends no `Origin` with a link, an image or a navigation, but + /// it does send `Sec-Fetch-Site`; only this server's own pages and the + /// user's own typing get through. + #[tokio::test(flavor = "multi_thread")] + async fn origin_less_browser_requests_must_come_from_the_docs_pages() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-fetch-site"); + services_dir_ready(&paths)?; + let model_port = start_echo_model_server().await; + record(&paths, "svc-live", "vllm", model_port).write()?; + let server = start(&paths)?; + let url = format!("{}/services/svc-live/api/v1/models", server.base_url); + let client = test_client(); + + for (site, expected) in [ + (Some("cross-site"), StatusCode::FORBIDDEN), + (Some("same-site"), StatusCode::FORBIDDEN), + (Some("same-origin"), StatusCode::OK), + (Some("none"), StatusCode::OK), + // Not a browser: curl and other programs send neither header. + (None, StatusCode::OK), + ] { + let mut request = client.get(&url); + if let Some(site) = site { + request = request.header(SEC_FETCH_SITE, site); + } + assert_eq!(request.send().await?.status(), expected, "{site:?}"); + } + + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn no_other_site_can_frame_the_docs() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-frames"); + services_dir_ready(&paths)?; + let model_port = start_echo_model_server().await; + record(&paths, "svc-live", "vllm", model_port).write()?; + let server = start(&paths)?; + let base = server.base_url.clone(); + let client = test_client(); + + for (path, csp) in [ + ("/", PAGE_CSP), + ("/services/svc-live/docs/", PAGE_CSP), + ("/services/svc-live/docs/swagger-initializer.js", PAGE_CSP), + ("/services/svc-live/openapi.json", PAGE_CSP), + ("/services/not-running/docs/", PAGE_CSP), + ("/services/svc-live/api/v1/models", FORWARDED_CSP), + ] { + let response = client.get(format!("{base}{path}")).send().await?; + let headers = response.headers(); + assert_eq!(headers[header::CONTENT_SECURITY_POLICY], csp, "{path}"); + assert_eq!(headers[header::X_FRAME_OPTIONS], "DENY", "{path}"); + assert_eq!(headers[header::X_CONTENT_TYPE_OPTIONS], "nosniff", "{path}"); + } + let refused = client + .get(format!("{base}/")) + .header(header::HOST, "attacker.example") + .send() + .await?; + assert_eq!(refused.status(), StatusCode::FORBIDDEN); + assert_eq!(refused.headers()[header::X_FRAME_OPTIONS], "DENY"); + + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn try_it_keeps_the_path_encoded_and_sends_large_bodies() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-forward-path"); + services_dir_ready(&paths)?; + let model_port = start_echo_model_server().await; + record(&paths, "svc-live", "vllm", model_port).write()?; + let server = start(&paths)?; + let base = server.base_url.clone(); + let client = test_client(); + + // Decoded, `%2F` would add a segment and `%3F` would start a query. + let forwarded: Value = client + .get(format!( + "{base}/services/svc-live/api/v1/models/Qwen%2FQwen3.5%3Fx" + )) + .send() + .await? + .text() + .await? + .parse()?; + assert_eq!(forwarded["path"], "/v1/models/Qwen%2FQwen3.5%3Fx"); + assert_eq!(forwarded["query"], Value::Null); + + // Larger than axum's 2 MiB default, as an audio upload can be. + let large = 3 * 1024 * 1024; + let response = client + .post(format!( + "{base}/services/svc-live/api/v1/audio/transcriptions" + )) + .body(vec![b'a'; large]) + .send() + .await?; + assert_eq!(response.status(), StatusCode::OK); + let forwarded: Value = response.text().await?.parse()?; + assert_eq!(forwarded["body_len"], large); + + // A redirect is passed back, not followed by the forwarder. + let redirect = client + .get(format!("{base}/services/svc-live/api/redirect")) + .send() + .await?; + assert_eq!(redirect.status(), StatusCode::TEMPORARY_REDIRECT); + + server.stop(); + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn ids_are_escaped_in_pages_and_encoded_in_links() -> Result<()> { + let (root, paths) = temp_app_paths("api-docs-escaping"); + services_dir_ready(&paths)?; + let model_port = start_echo_model_server().await; + let odd_id = "svc a#1%"; + let mut odd = record(&paths, odd_id, "vllm", model_port); + odd.canonical_model_id = "model".to_owned(); + odd.write()?; + let server = start(&paths)?; + let base = server.base_url.clone(); + let client = test_client(); + + let link = server.service_docs_url(odd_id); + assert_eq!(link, format!("{base}/services/svc%20a%231%25/docs/")); + let page = client.get(&link).send().await?; + assert_eq!(page.status(), StatusCode::OK, "the printed link opens"); + let initializer = client + .get(format!("{link}swagger-initializer.js")) + .send() + .await? + .text() + .await?; + assert!( + initializer.contains("/services/svc%20a%231%25/openapi.json"), + "{initializer}" + ); + let openapi = client + .get(format!("{base}/services/svc%20a%231%25/openapi.json")) + .send() + .await?; + assert_eq!(openapi.status(), StatusCode::OK); + let openapi: Value = openapi.text().await?.parse()?; + assert_eq!(openapi["servers"][0]["url"], "/services/svc%20a%231%25/api"); + let redirect = client + .get(format!("{base}/services/svc%20a%231%25/docs")) + .send() + .await?; + assert_eq!( + redirect.headers()[header::LOCATION], + "/services/svc%20a%231%25/docs/" + ); + + let index = client.get(format!("{base}/")).send().await?.text().await?; + assert!( + index.contains("href=\"/services/svc%20a%231%25/docs/\""), + "{index}" + ); + assert!(index.contains("<b>model</b>"), "{index}"); + assert!(!index.contains(""), "{index}"); + + // The id of a server that is not running is reflected into the page. + let crafted = client + .get(format!( + "{base}/services/%3Cscript%3Ealert(1)%3C%2Fscript%3E/docs/" + )) + .send() + .await?; + assert_eq!(crafted.status(), StatusCode::NOT_FOUND); + let crafted = crafted.text().await?; + assert!(crafted.contains("<script>alert(1)"), "{crafted}"); + assert!(!crafted.contains("