diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 96b63f1b9..47903ebd5 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -471,10 +471,11 @@ jobs: # this replaced, and a run took 26m54s against the old 35min cap (run # 34489547654) versus 18-19m on a warm static host — keep the same headroom # ratio the 35 was chosen for. - # An include_nightly dispatch also runs the AIM serve scenarios (serve-26/27, - # @nightly, up to 1200s readiness each on a cold pull) and the large-model - # serve, which cannot fit the 45min budget of an ordinary push run. - timeout-minutes: ${{ inputs.include_nightly && 105 || 45 }} + # An include_nightly dispatch also runs the AIM serve scenarios + # (serve-26/27/62, @nightly, up to 1200s readiness each on a cold pull) and + # the large-model serve, which cannot fit the 45min budget of an ordinary + # push run. + timeout-minutes: ${{ inputs.include_nightly && 125 || 45 }} runs-on: [self-hosted, linux, devlab-dispatch, strix-halo] needs: [changes] # See `e2e-gpu`: no build-and-test gate (cross-workflow); strix-ubuntu. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7aca0e072..bea64ac5c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -512,11 +512,13 @@ jobs: # Strix Ubuntu job. e2e-gpu-nightly-strix: name: E2E tests (Strix Halo, incl. nightly-only) - # 120: serve-26/27 (AIM, @nightly) add up to 2 x 1200s readiness waits; on a - # host without Docker they skip and cost nothing. The "Report whether Docker - # is reachable" step logs the answer each run. serve-27's known failure on - # this pool is an xfail row in expectations.toml (rocm-cli#1058). - timeout-minutes: 120 + # 140: serve-26/27/62 (AIM, @nightly) add up to 3 x 1200s readiness waits + # (each container downloads its checkpoint afresh: none is cached between + # scenarios); on a host without Docker they skip and cost nothing. The + # "Report whether Docker is reachable" step logs the answer each run. + # serve-27 once failed here (rocm-cli#1058) and now runs expect-pass, with + # no row in expectations.toml. + timeout-minutes: 140 # The DevLab Dispatch pool, identical to the e2e-gpu-strix-ubuntu lane in # e2e-selfhosted.yml, which runs on push and dispatch but not on pull_request # or merge_group (pinned by workflow_contract.rs). This lane used to target @@ -563,14 +565,14 @@ jobs: fi echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - # serve-26/27 (AIM) need Docker reachable on this host; without it they skip + # serve-26/27/62 (AIM) need Docker reachable on this host; without it they skip # silently. Say which it is, so a nightly that never ran them is visible. - name: Report whether Docker is reachable (AIM serve scenarios need it) run: | if docker info >/dev/null 2>&1; then - echo "docker reachable: serve-26/27 (AIM) can run on this host" + echo "docker reachable: serve-26/27/62 (AIM) can run on this host" else - echo "::warning::docker is not reachable on this host: serve-26/27 (AIM) will skip" + echo "::warning::docker is not reachable on this host: serve-26/27/62 (AIM) will skip" fi - name: Run E2E tests on Strix Halo (incl. nightly-only) diff --git a/Cargo.lock b/Cargo.lock index ed1f160cf..a503c9c35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3911,6 +3911,7 @@ dependencies = [ "rocm-core", "serde", "serde_json", + "tempfile", ] [[package]] diff --git a/README.md b/README.md index f5ffc7b4e..02bd6cbb2 100644 --- a/README.md +++ b/README.md @@ -1313,6 +1313,26 @@ directory that only your user account can read. Set `ROCM_SERVE_API_KEY` rather passing `--api-key`, so the secret stays out of your shell history and the process table. +With the vLLM and AIM engines, an endpoint bound to a public interface +(`--allow-public-bind`), or served with `--require-api-key`, answers only +clients that present its API key as an `Authorization: Bearer` header, on every +route the server answers, not just the OpenAI `/v1` API — including vLLM's own +`/invocations`, `/tokenize` and `/metrics` — except `GET`/`HEAD /health` and +CORS preflights: the `/health` liveness check stays open. An AIM container gets +this check through `AIM_ENGINE_ARGS`; one whose image does not apply it fails to +start and its container is removed, rather than serving with only `/v1` +protected. A key-protected AIM launch also refuses to start when it cannot read +the image's config, or when `AIM_ENGINE_ARGS` (from `--engine-env` or the image) +is not a JSON object or already names a `middleware`. + +Because `/metrics` is covered, `rocm dash` cannot read engine metrics from a +key-protected vLLM server: it leaves that server's KV-cache, queue, throughput +and latency figures empty and keeps showing a warning such as +`vllm scrape: 1 failed (last: transport: GET http://:/metrics: status 401 Unauthorized)`. + +`rocm bench load` sends no API key, so it cannot benchmark a key-protected +endpoint on any engine. + `--temperature` (>= 0.0), `--top-p` (0.0-1.0), and `--max-tokens` (> 0) set server-wide sampling defaults for the launched engine. They apply only to `vllm` and `lemonade`; other engines reject them. For vLLM they are folded diff --git a/apps/rocm/src/service_launch/endpoint_auth.rs b/apps/rocm/src/service_launch/endpoint_auth.rs index 6a647314b..b14cf76fb 100644 --- a/apps/rocm/src/service_launch/endpoint_auth.rs +++ b/apps/rocm/src/service_launch/endpoint_auth.rs @@ -98,9 +98,12 @@ pub(crate) fn drop_orphaned_endpoint_key_on_already_running( /// The one such case today: Windows managed Lemonade. Its server reads the /// value-typed `LEMONADE_API_KEY` env var, but the Windows detached-spawn /// primitive only carries path-valued env overrides, so the key never reaches it. -/// vLLM enforces auth on every platform (`VLLM_API_KEY`), and loopback binds -/// (`public_bind == false`) need no key — both pass. `is_windows` is a parameter -/// so both branches are unit-testable off-Windows. +/// The vllm and aim engines enforce auth wherever they run (`VLLM_API_KEY` plus +/// the every-route middleware in `crates/rocm-engine-protocol/src/endpoint_auth.rs`, +/// which an AIM serve through `serve_http` — the path `rocm serve` takes — +/// verifies before serving), and loopback binds (`public_bind == false`) need no +/// key — both pass. `is_windows` is a parameter so both branches are +/// unit-testable off-Windows. pub(crate) fn ensure_public_bind_engine_supported( engine: &str, public_bind: bool, diff --git a/crates/rocm-engine-protocol/Cargo.toml b/crates/rocm-engine-protocol/Cargo.toml index d9f9fa82b..1f7c81023 100644 --- a/crates/rocm-engine-protocol/Cargo.toml +++ b/crates/rocm-engine-protocol/Cargo.toml @@ -20,6 +20,7 @@ anyhow.workspace = true rocm-core = { path = "../rocm-core" } serde.workspace = true serde_json.workspace = true +tempfile = "3" [dev-dependencies] # The leftover a killed owner-only key store leaves, for the tests of the diff --git a/crates/rocm-engine-protocol/src/endpoint_auth.rs b/crates/rocm-engine-protocol/src/endpoint_auth.rs new file mode 100644 index 000000000..ea6480acf --- /dev/null +++ b/crates/rocm-engine-protocol/src/endpoint_auth.rs @@ -0,0 +1,360 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The endpoint-key middleware for vLLM's OpenAI server, shared by every engine +//! that runs one. +//! +//! Setting `VLLM_API_KEY` alone does not protect a vLLM endpoint: vLLM's own +//! check guards only a fixed set of path prefixes (`/v1` and a few others), +//! while the server also registers inference-capable routes outside them — +//! `/invocations`, `/tokenize`, `/detokenize` and more. So whenever an endpoint +//! key is in force, the server is additionally started with +//! `--middleware rocm_cli_vllm_auth.RequireEndpointApiKey`: the small ASGI +//! middleware in `rocm_cli_vllm_auth.py` beside this file, compiled +//! into the binary and written out at launch, that requires the key on every +//! route except `GET`/`HEAD /health` and CORS preflights. +//! +//! This module owns the middleware itself — its source, its import path and +//! writing it to disk. How it reaches the server is each engine's business: +//! the native vLLM engine puts the directory on the server's `PYTHONPATH` +//! (`engines/vllm`'s `endpoint_auth.rs`), while the AIM engine bind-mounts it +//! into the container (`engines/aim`'s `endpoint_auth.rs`). + +use anyhow::{Context, Result}; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +/// Python module name the middleware is imported as. +pub const MIDDLEWARE_MODULE_NAME: &str = "rocm_cli_vllm_auth"; + +/// Source of the middleware module, compiled into the binary so the copy vLLM +/// imports always matches the adapter that launched it. +pub const MIDDLEWARE_MODULE_SOURCE: &str = include_str!("rocm_cli_vllm_auth.py"); + +/// The `--middleware` import path vLLM is given. +pub const MIDDLEWARE_IMPORT_PATH: &str = "rocm_cli_vllm_auth.RequireEndpointApiKey"; + +/// Environment variable carrying the key to both vLLM's own check and the +/// middleware. The middleware reads this exact name (`KEY_ENV` in the module). +pub const MIDDLEWARE_KEY_ENV: &str = "VLLM_API_KEY"; + +/// Extra `vllm serve` arguments that install the middleware. +#[must_use] +pub fn middleware_args() -> [String; 2] { + ["--middleware".to_owned(), MIDDLEWARE_IMPORT_PATH.to_owned()] +} + +/// Write the middleware module into `dir`, replacing any earlier copy, and +/// return the file's path. +/// +/// Written to a temporary file and renamed into place, so a concurrent launch +/// importing the module never reads a half-written file. The file holds no +/// secret, and on unix it and `dir` are made world-readable (`0644`/`0755`): +/// a container that bind-mounts `dir` may import it as a user other than the +/// one who wrote it. +pub fn write_middleware_module(dir: &Path) -> Result { + fs::create_dir_all(dir).with_context(|| format!("failed to create {}", dir.display()))?; + let path = dir.join(format!("{MIDDLEWARE_MODULE_NAME}.py")); + let mut staged = tempfile::NamedTempFile::new_in(dir) + .with_context(|| format!("failed to stage a file in {}", dir.display()))?; + staged + .write_all(MIDDLEWARE_MODULE_SOURCE.as_bytes()) + .with_context(|| format!("failed to write {}", path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + staged + .as_file() + .set_permissions(fs::Permissions::from_mode(0o644)) + .with_context(|| format!("failed to set permissions on {}", path.display()))?; + fs::set_permissions(dir, fs::Permissions::from_mode(0o755)) + .with_context(|| format!("failed to set permissions on {}", dir.display()))?; + } + staged + .persist(&path) + .with_context(|| format!("failed to write {}", path.display()))?; + Ok(path) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::{Command, Stdio}; + + #[test] + fn write_middleware_module_replaces_a_stale_copy() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join(format!("{MIDDLEWARE_MODULE_NAME}.py")); + fs::write(&path, "class RequireEndpointApiKey: pass\n")?; + assert_eq!(write_middleware_module(temp.path())?, path); + assert_eq!(fs::read_to_string(&path)?, MIDDLEWARE_MODULE_SOURCE); + Ok(()) + } + + /// A launch that cannot write the module must fail naming the path, never + /// start a server without the middleware it asked for. + #[test] + fn write_middleware_module_fails_naming_a_directory_that_is_a_file() -> Result<()> { + let temp = tempfile::tempdir()?; + let dir = temp.path().join("python"); + fs::write(&dir, "not a directory")?; + let error = format!("{:#}", write_middleware_module(&dir).unwrap_err()); + assert!( + error.contains(&format!("failed to create {}", dir.display())), + "{error}" + ); + Ok(()) + } + + #[test] + fn write_middleware_module_fails_naming_a_module_path_taken_by_a_directory() -> Result<()> { + let temp = tempfile::tempdir()?; + let path = temp.path().join(format!("{MIDDLEWARE_MODULE_NAME}.py")); + fs::create_dir_all(path.join("occupied"))?; + let error = format!("{:#}", write_middleware_module(temp.path()).unwrap_err()); + assert!( + error.contains(&format!("failed to write {}", path.display())), + "{error}" + ); + Ok(()) + } + + /// Asserts nothing as root, which can create files in a read-only + /// directory: the probe write then succeeds and the check is skipped. + #[cfg(unix)] + #[test] + fn write_middleware_module_fails_naming_a_directory_it_cannot_write() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir()?; + let dir = temp.path().join("python"); + fs::create_dir_all(&dir)?; + fs::set_permissions(&dir, fs::Permissions::from_mode(0o500))?; + if fs::write(dir.join("probe"), "").is_err() { + let result = write_middleware_module(&dir); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o700))?; + let error = format!("{:#}", result.unwrap_err()); + assert!( + error.contains(&format!("failed to stage a file in {}", dir.display())), + "{error}" + ); + } + Ok(()) + } + + /// A container bind-mounting the directory may run as another user, so + /// both the directory and the module must be readable by anyone. + #[cfg(unix)] + #[test] + fn write_middleware_module_leaves_the_module_readable_by_other_users() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir()?; + let dir = temp.path().join("python"); + fs::create_dir_all(&dir)?; + fs::set_permissions(&dir, fs::Permissions::from_mode(0o700))?; + let path = write_middleware_module(&dir)?; + assert_eq!(fs::metadata(&path)?.permissions().mode() & 0o777, 0o644); + assert_eq!(fs::metadata(&dir)?.permissions().mode() & 0o777, 0o755); + Ok(()) + } + + #[test] + fn import_path_names_the_shipped_module_and_class() { + let (module, class) = MIDDLEWARE_IMPORT_PATH + .rsplit_once('.') + .expect("module.Class"); + assert_eq!(module, MIDDLEWARE_MODULE_NAME); + assert!(MIDDLEWARE_MODULE_SOURCE.contains(&format!("\nclass {class}:"))); + assert!(MIDDLEWARE_MODULE_SOURCE.contains(&format!("KEY_ENV = \"{MIDDLEWARE_KEY_ENV}\""))); + } + + #[test] + fn middleware_args_install_the_shipped_class() { + assert_eq!( + middleware_args(), + ["--middleware".to_owned(), MIDDLEWARE_IMPORT_PATH.to_owned()] + ); + } + + /// Drives the shipped middleware with a stdlib-only ASGI harness, the same + /// way vLLM's `--middleware` loader mounts it (`app.add_middleware(cls)` + /// wraps the app in `cls(app)`). Proves the routes vLLM leaves outside its + /// own key check — `/invocations`, `/tokenize`, ... — answer 401 without + /// the key, and that only `GET`/`HEAD /health` and CORS preflights stay + /// open. + /// + /// Needs `python3` on PATH and fails loudly without it rather than passing + /// vacuously. Linux-only: vLLM serving (natively or in an AIM container) is + /// Linux/WSL-only in rocm-cli, so the module never runs anywhere else. + #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "vLLM serving (and so this middleware) is Linux/WSL-only" + )] + fn middleware_requires_the_key_on_every_route_but_health() -> Result<()> { + let temp = tempfile::tempdir()?; + let module_dir = temp.path().join("python"); + write_middleware_module(&module_dir)?; + let harness = temp.path().join("harness.py"); + fs::write(&harness, ASGI_HARNESS)?; + + let output = Command::new("python3") + .arg(&harness) + .env("PYTHONPATH", &module_dir) + .env(MIDDLEWARE_KEY_ENV, "secret-key") + .stdin(Stdio::null()) + .output() + .context("python3 is required to exercise the vLLM auth middleware")?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "middleware harness failed\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + assert!(stdout.contains("ALL OK"), "harness stdout:\n{stdout}"); + Ok(()) + } + + /// Construction must fail when the key is empty, so a protected launch + /// whose key went missing cannot answer a request without checking one. + #[test] + #[cfg_attr( + not(target_os = "linux"), + ignore = "vLLM serving (and so this middleware) is Linux/WSL-only" + )] + fn middleware_refuses_to_start_without_a_key() -> Result<()> { + let temp = tempfile::tempdir()?; + write_middleware_module(temp.path())?; + let output = Command::new("python3") + .args([ + "-c", + "import rocm_cli_vllm_auth as m; m.RequireEndpointApiKey(lambda *a: None)", + ]) + .env("PYTHONPATH", temp.path()) + .env_remove(MIDDLEWARE_KEY_ENV) + .stdin(Stdio::null()) + .output() + .context("python3 is required to exercise the vLLM auth middleware")?; + assert!( + !output.status.success(), + "constructing without a key must fail" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("VLLM_API_KEY is empty"), "stderr: {stderr}"); + Ok(()) + } + + const ASGI_HARNESS: &str = r#" +import asyncio +import rocm_cli_vllm_auth as m + +async def inner(scope, receive, send): + if scope["type"] == "websocket": + await send({"type": "websocket.accept"}) + return + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"inner"}) + +async def call(app, method, path, headers=(), kind="http", root_path=""): + sent = [] + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + async def send(message): + sent.append(message) + scope = {"type": kind, "method": method, "path": path, "root_path": root_path, + "headers": [(k.lower().encode(), v.encode()) for k, v in headers]} + await app(scope, receive, send) + first = sent[0] + if first["type"] == "http.response.start": + return first["status"] + return first["type"] + +GOOD = [("Authorization", "Bearer secret-key")] +CASES = [ + # Routes outside vLLM's own guarded prefixes must need the key too. + ("POST", "/invocations", (), 401), + ("POST", "/tokenize", (), 401), + ("POST", "/detokenize", (), 401), + ("POST", "/generative_scoring", (), 401), + ("POST", "/scale_elastic_ep", (), 401), + ("GET", "/metrics", (), 401), + ("GET", "/ping", (), 401), + ("POST", "/v1/chat/completions", (), 401), + ("GET", "/some/future/route", (), 401), + # Wrong, malformed or non-bearer credentials. + ("POST", "/invocations", [("Authorization", "Bearer wrong")], 401), + ("POST", "/invocations", [("Authorization", "Bearer secret-key-and-more")], 401), + ("POST", "/invocations", [("Authorization", "Basic secret-key")], 401), + ("POST", "/invocations", [("Authorization", "secret-key")], 401), + ("POST", "/invocations", [("X-Api-Key", "secret-key")], 401), + # Only GET/HEAD /health is open, and only at exactly that path. + ("GET", "/health", (), 200), + ("HEAD", "/health", (), 200), + ("POST", "/health", (), 401), + ("GET", "/health/", (), 401), + ("GET", "/healthz", (), 401), + # A CORS preflight (OPTIONS carrying both Origin and + # Access-Control-Request-Method) is the only other keyless request. Each + # part alone is not one, and neither are those headers on another method. + ("OPTIONS", "/invocations", (), 401), + ("OPTIONS", "/invocations", + [("Origin", "http://x"), ("Access-Control-Request-Method", "POST")], 200), + ("OPTIONS", "/invocations", [("Origin", "http://x")], 401), + ("OPTIONS", "/invocations", [("Access-Control-Request-Method", "POST")], 401), + ("POST", "/invocations", + [("Origin", "http://x"), ("Access-Control-Request-Method", "POST")], 401), + # The right key opens every route, with any bearer-scheme casing. + ("POST", "/invocations", GOOD, 200), + ("POST", "/tokenize", GOOD, 200), + ("GET", "/metrics", GOOD, 200), + ("POST", "/v1/chat/completions", [("Authorization", "bearer secret-key")], 200), +] + +async def main(): + app = m.RequireEndpointApiKey(inner) + failures = [] + for method, path, headers, want in CASES: + got = await call(app, method, path, headers) + if got != want: + failures.append(f"{method} {path} {headers!r}: want {want}, got {got}") + # root_path is stripped before matching, as vLLM's own check does. + got = await call(app, "GET", "/prefix/health", root_path="/prefix") + if got != 200: + failures.append(f"root_path /health: want 200, got {got}") + # Only a path that starts with root_path has it stripped. A path of the same + # length as root_path but outside it is not the health check, and neither is + # a path that holds root_path anywhere but at its start. + for path in ("/xxxxxx/health", "/health/prefix"): + got = await call(app, "GET", path, root_path="/prefix") + if got != 401: + failures.append(f"{path} outside root_path: want 401, got {got}") + # WebSocket routes are closed before the handshake without the key. + got = await call(app, None, "/v1/realtime", kind="websocket") + if got != "websocket.close": + failures.append(f"websocket without key: want close, got {got}") + got = await call(app, None, "/v1/realtime", GOOD, kind="websocket") + if got != "websocket.accept": + failures.append(f"websocket with key: want accept, got {got}") + # The /health exemption is for HTTP GET/HEAD only, not a WebSocket there. + # This catches an exemption keyed on the path alone (for any scope type). + # Dropping only the scope-type guard in `_exempt` would still pass here, + # because a WebSocket scope carries no method for the GET/HEAD check. + got = await call(app, None, "/health", kind="websocket") + if got != "websocket.close": + failures.append(f"websocket /health without key: want close, got {got}") + # Lifespan events pass straight through. + seen = [] + async def lifespan_inner(scope, receive, send): + seen.append(scope["type"]) + await m.RequireEndpointApiKey(lifespan_inner)({"type": "lifespan"}, None, None) + if seen != ["lifespan"]: + failures.append(f"lifespan not passed through: {seen}") + if failures: + print("\n".join(failures)) + raise SystemExit(1) + print("ALL OK") + +asyncio.run(main()) +"#; +} diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index e13731c68..03c36fef9 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -18,6 +18,8 @@ use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; +pub mod endpoint_auth; + pub const ENGINE_PROTOCOL_VERSION: &str = "0.1.0"; pub const ENGINE_RECIPE_CONTRACT_VERSION: &str = "0.1.0"; pub const ENGINE_PLUGIN_BINARY_PREFIX: &str = "rocm-engine-"; diff --git a/crates/rocm-engine-protocol/src/rocm_cli_vllm_auth.py b/crates/rocm-engine-protocol/src/rocm_cli_vllm_auth.py new file mode 100644 index 000000000..7d41c49c9 --- /dev/null +++ b/crates/rocm-engine-protocol/src/rocm_cli_vllm_auth.py @@ -0,0 +1,126 @@ +# Copyright © Advanced Micro Devices, Inc., or its affiliates. +# +# SPDX-License-Identifier: MIT + +"""Require the endpoint API key on every route of a vLLM OpenAI server. + +rocm-cli passes ``--middleware rocm_cli_vllm_auth.RequireEndpointApiKey`` to +``vllm serve`` -- directly, or through ``AIM_ENGINE_ARGS`` in an AIM +container -- whenever it protects an endpoint with an API key. vLLM's own +``VLLM_API_KEY`` check guards only a fixed set of path prefixes (``/v1`` and +a few others), while the server also registers inference-capable routes +outside them (``/invocations``, ``/tokenize``, ...). This middleware closes +that gap by checking the key on every HTTP and WebSocket request, whatever +its path, so a route vLLM adds in a later release is covered too. + +Only ``GET``/``HEAD /health`` and CORS preflight requests pass without the +key: the first is the liveness probe rocm-cli advertises, the second carries +no credentials by design and is answered by vLLM's CORS middleware before +any route runs. + +The key is read from ``VLLM_API_KEY`` -- the same variable vLLM's own check +reads, set by rocm-cli on the server process -- so both checks always agree. +Construction fails when the variable is empty, so a launch that asked for +protection cannot come up unprotected. + +Pure ASGI and standard library only: it must import in whichever vLLM +release the runtime ships, with no assumption about its Starlette version. +This file is written out by the rocm-cli vLLM and AIM engine adapters at +launch; edit the copy in the rocm-cli source tree, not the one on disk. +""" + +import hashlib +import hmac +import os + +KEY_ENV = "VLLM_API_KEY" + +# Exact paths (after any ASGI root_path) that answer without the key, and the +# only methods they answer to. Everything else requires the key. +UNAUTHENTICATED_ROUTES = {"/health": frozenset({"GET", "HEAD"})} + +_UNAUTHORIZED_BODY = b'{"error":"Unauthorized"}' + +# RFC 6455 policy-violation close code. Closing before accepting makes the +# server reject the WebSocket handshake with HTTP 403. +_WS_POLICY_VIOLATION = 1008 + + +def _digest(value): + return hashlib.sha256(value).digest() + + +class RequireEndpointApiKey: + """ASGI middleware rejecting any request that lacks the bearer key.""" + + def __init__(self, app): + key = os.environ.get(KEY_ENV, "") + if not key: + raise RuntimeError( + f"{KEY_ENV} is empty; refusing to start an endpoint that was " + "asked to require an API key without one" + ) + self.app = app + self._expected = _digest(key.encode("utf-8")) + + def _authorized(self, scope): + authorization = None + for name, value in scope.get("headers") or (): + if name.lower() == b"authorization": + authorization = value + break + if authorization is None: + return False + scheme, _, token = authorization.decode("latin-1").partition(" ") + if scheme.lower() != "bearer": + return False + # Compare fixed-length digests so the comparison time depends on + # neither the key's length nor how much of it matched. + return hmac.compare_digest(_digest(token.encode("utf-8")), self._expected) + + @staticmethod + def _route_path(scope): + path = scope.get("path", "") + root_path = scope.get("root_path", "") + if root_path and path.startswith(root_path): + path = path[len(root_path) :] + return path + + @staticmethod + def _is_cors_preflight(scope): + if scope.get("method") != "OPTIONS": + return False + names = {name.lower() for name, _ in scope.get("headers") or ()} + return b"origin" in names and b"access-control-request-method" in names + + def _exempt(self, scope): + if scope["type"] != "http": + return False + if self._is_cors_preflight(scope): + return True + methods = UNAUTHENTICATED_ROUTES.get(self._route_path(scope)) + return methods is not None and scope.get("method") in methods + + async def __call__(self, scope, receive, send): + kind = scope["type"] + if kind not in ("http", "websocket") or self._exempt(scope): + await self.app(scope, receive, send) + return + if self._authorized(scope): + await self.app(scope, receive, send) + return + if kind == "websocket": + await send({"type": "websocket.close", "code": _WS_POLICY_VIOLATION}) + return + await send( + { + "type": "http.response.start", + "status": 401, + "headers": [ + (b"content-type", b"application/json"), + (b"content-length", str(len(_UNAUTHORIZED_BODY)).encode()), + (b"www-authenticate", b"Bearer"), + ], + } + ) + await send({"type": "http.response.body", "body": _UNAUTHORIZED_BODY}) diff --git a/docs/architecture.md b/docs/architecture.md index 75d87bf0e..4994ee929 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -614,6 +614,7 @@ A contract surface: verify all impacted engines after any protocol change here ( | File | Responsibility | Class | | --- | --- | --- | | [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) | The protocol contract: protocol versions, the request/response envelopes and per-method types, device and GPU-selection policy, engine-plugin discovery, the endpoint API key file carrier shared by `rocm` and `rocmd` (including `remove_endpoint_key_files`, which also removes leftover staged copies of a key), and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. | flat | +| [`endpoint_auth.rs`](../crates/rocm-engine-protocol/src/endpoint_auth.rs) | The every-route endpoint-key middleware for vLLM's OpenAI server, shared by the two engines that run one (native vLLM and AIM's containers): the source of [`rocm_cli_vllm_auth.py`](../crates/rocm-engine-protocol/src/rocm_cli_vllm_auth.py) (compiled in with `include_str!`), its `--middleware` import path, and `write_middleware_module`, which writes it world-readable so a container can import it. `VLLM_API_KEY` alone covers only a fixed set of path prefixes; the middleware requires the bearer key on every HTTP and WebSocket route except `GET`/`HEAD /health` and CORS preflights. How it reaches the server is each engine's own `endpoint_auth.rs`. | flat | | [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) | Test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. | flat | | [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) | The client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, whether to withhold an inherited key file from a child that is handed none, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written (the error then carries what it wrote to stderr) or its answer cannot be read, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. It sets no timeout, and does not bound the wait for the child's pipes after it exits (a grandchild holding one makes the request wait); its module comment states that limit. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). | flat | @@ -691,6 +692,7 @@ The vLLM engine adapter (`rocm-engine-vllm`): installs vLLM into a managed TheRo | [`lib.rs`](../engines/vllm/src/lib.rs) | Crate root: CLI and envelope dispatch and the few items shared across the modules, including `verified_torchaudio_release` for the `rocm` binary, a `pub fn` here that delegates to the `pub(crate)` one in [`wheel_builds.rs`](../engines/vllm/src/wheel_builds.rs) (it is not a `pub use`). | flat | | [`main.rs`](../engines/vllm/src/main.rs) | The `rocm-engine-vllm` binary's entry point; it only calls `rocm_engine_vllm::run_cli()`. | flat | | [`discover_builds.rs`](../engines/vllm/src/discover_builds.rs) | The ROCm 10.x discovery table (`VLLM_ROCM_DISCOVER_BUILD_TABLE`, one row per supported `major.minor` line), its lookup and the series matcher. | domain | +| [`endpoint_auth.rs`](../engines/vllm/src/endpoint_auth.rs) | How a keyed launch installs the shared endpoint-key middleware (`rocm_engine_protocol::endpoint_auth`): writes it under the engine's data directory, sets `VLLM_API_KEY` and prepends that directory to the server's `PYTHONPATH`, keeping the value the command or the environment would otherwise have given it; `vllm_serve_args` adds `--middleware rocm_cli_vllm_auth.RequireEndpointApiKey`. | domain | | [`install.rs`](../engines/vllm/src/install.rs) | ROCm and vLLM install execution through `uv`, by the static-pin route or the ROCm 10.x discovery route, then a check that vLLM's `tensorizer` imports (upgrading it when it does not). | domain | | [`process.rs`](../engines/vllm/src/process.rs) | The resolve-model, launch, serve and stop handlers: recipe validation, device-policy and GPU-index resolution, the vLLM server spawn and launch environment, and readiness polling. | domain | | [`repair.rs`](../engines/vllm/src/repair.rs) | Whether an already-resolvable managed vLLM environment violates vLLM's pinned dependencies and must be reinstalled (`assess_runtime_repair`), treating the SDK's torch build, a user-chosen torch stack (torch, torchvision, torchaudio, triton) with alignment disabled, and the tensorizer upgrade as intended. | domain | @@ -715,6 +717,7 @@ The AIM engine adapter (`rocm-engine-aim`) runs models as Docker containers inst | [`lib.rs`](../engines/aim/src/lib.rs) | Crate root: CLI and envelope dispatch, including the `builtin_*` entry points the `rocm` binary calls in-process, the detect and capabilities responses, and the items shared across the modules. | flat | | [`main.rs`](../engines/aim/src/main.rs) | The `rocm-engine-aim` binary's entry point; it only calls `rocm_engine_aim::run_cli()`. | flat | | [`docker.rs`](../engines/aim/src/docker.rs) | A thin wrapper around the `docker` CLI binary, with no Docker API client, and `container_name`, a service's deterministic Docker-safe container name. | flat | +| [`endpoint_auth.rs`](../engines/aim/src/endpoint_auth.rs) | How a keyed AIM container gets the shared every-route endpoint-key middleware (`rocm_engine_protocol::endpoint_auth`): the module is bind-mounted read-only at `/opt/rocm-cli/python`, and the container's env file gets `VLLM_API_KEY`, that directory prepended to `PYTHONPATH` and `"middleware"` added to `AIM_ENGINE_ARGS`, keeping the caller's or the image's values. A keyed launch is refused when the image's config cannot be read or its `AIM_ENGINE_ARGS` cannot carry the middleware, with a remedy that depends on where that value came from. `verify_every_route_guarded` is the check `serve_http` runs once a keyed container is ready: a keyless `POST /tokenize` must answer `401`, or the start fails. | flat | | [`install.rs`](../engines/aim/src/install.rs) | Install for an engine whose images are self-contained: only a check that Docker is present and reachable. | flat | | [`process.rs`](../engines/aim/src/process.rs) | The container run, readiness and stop lifecycle, including `serve_http`, which blocks for the container's lifetime and records how it ended. | flat | | [`registry.rs`](../engines/aim/src/registry.rs) | AIM image metadata (`Env`/`Labels`), read from the local daemon once the image is pulled, with a best-effort Docker Hub v2 lookup as a pre-pull hint. | flat | diff --git a/docs/manual-testing.md b/docs/manual-testing.md index f7bab1843..3de0af081 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -1481,7 +1481,38 @@ On some GPU families `rocm install sdk` installs vLLM itself the `rocm engines install vllm` that follows then shows the block without the warning, and nothing more needs doing. -## 7. ComfyUI Verification +## 7. vLLM Endpoint Key Coverage + +Needs a Linux or WSL host where the vLLM engine can start. Check that an +endpoint key covers every route the vLLM server answers, not only `/v1`: + +```bash +export ROCM_SERVE_API_KEY=manual-test-key +rocm serve Qwen/Qwen3.5-0.8B --engine vllm --require-api-key --managed +PORT= +curl -s -o /dev/null -w '%{http_code}\n' -X POST "http://127.0.0.1:$PORT/invocations" \ + -H 'Content-Type: application/json' \ + -d '{"model":"Qwen/Qwen3.5-0.8B","messages":[{"role":"user","content":"hi"}]}' +curl -s -o /dev/null -w '%{http_code}\n' -X POST "http://127.0.0.1:$PORT/tokenize" \ + -H 'Content-Type: application/json' -d '{"model":"Qwen/Qwen3.5-0.8B","prompt":"hi"}' +curl -s -o /dev/null -w '%{http_code}\n' "http://127.0.0.1:$PORT/health" +``` + +Expected result: + +- The `/invocations` and `/tokenize` calls print `401`. +- Repeating them with `-H "Authorization: Bearer $ROCM_SERVE_API_KEY"` prints + `200`. +- The `/health` call prints `200` without the key. `GET`/`HEAD /health` and + CORS preflights are the only requests the server answers without it. + +Stop the service afterwards (`rocm services stop --yes`). The +automated scenario is the `@nightly` `serve-vllm-key-guards-every-route` in +[model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature), +which runs after merge in the nightly runs on Instinct GPUs (MI300X, and MI350P +on the shared GPU queue). + +## 8. ComfyUI Verification ComfyUI is managed as an app surface. It should start a local web server and show the URL to open: @@ -1560,7 +1591,7 @@ python scripts\comfyui_therock_gpu_test.py This test may download a small checkpoint and submit a cat image workflow through the ComfyUI HTTP API. -## 8. Optional Cloud Provider Key +## 9. Optional Cloud Provider Key Local ROCm use does not need a cloud provider key. If you want to test OpenAI or Anthropic provider setup, save the key through stdin so it does not land in @@ -1585,7 +1616,7 @@ To remove the saved key: rocm config clear-provider-key openai ``` -## 9. Optional Provider-Assisted Planning +## 10. Optional Provider-Assisted Planning Most users should leave this off. To test ambiguity resolution with an already running local provider service: @@ -1613,7 +1644,7 @@ To turn provider-assisted planning back off: rocm config clear-planner-provider ``` -## 10. Uninstall +## 11. Uninstall Run this last, on a machine installed with the release installer (not a `cargo` build, whose binaries uninstall leaves alone): @@ -1689,7 +1720,7 @@ Expected result: either case. The same prompt is shared by `rocm fix` and the SDK-overwrite prompt, so the same answers behave the same way there. -## 11. AIM (Docker) Engine Verification +## 12. AIM (Docker) Engine Verification Linux/WSL2 only, and requires a reachable Docker daemon (`docker info` must succeed). The catalog entry used below has profiles for Radeon PRO W7900 and @@ -1754,7 +1785,34 @@ Expected result: - On native Windows, the same command fails explicitly instead of falling back to another engine. -## 11. Background Automation Helper +To check that an endpoint key covers every route of an AIM container, not only +`/v1`, serve the same raw image behind a key and repeat the calls from section +7 against it: + +```bash +export ROCM_SERVE_API_KEY=manual-test-key +rocm serve amdenterpriseai/aim-radeon-qwen-qwen3-5-9b:0.12.0-preview --engine aim \ + --managed --engine-env AIM_PROFILE_ID=vllm-w7900-bf16-tp1-latency --require-api-key +docker inspect --format '{{json .Mounts}}' rocm-aim- +``` + +Expected result: + +- The service reaches `ready`, and `docker inspect` shows the engine's + `python` directory mounted read-only at `/opt/rocm-cli/python`. +- With `Qwen/Qwen3.5-9B` as the model, the `/invocations` and `/tokenize` calls + from section 7 print `401` without the key and `200` with + `-H "Authorization: Bearer $ROCM_SERVE_API_KEY"`; `/health` prints `200` + without it. +- If the image does not apply `AIM_ENGINE_ARGS`, the serve fails instead, + naming `POST /tokenize` and the status it answered, and the container is + removed. + +The automated scenario is the `@nightly` `serve-aim-key-guards-every-route` in +[model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature), +which runs only on a nightly host with Docker and a Radeon or Strix Halo GPU. + +## 12. Background Automation Helper The helper (`rocm daemon`) runs only while an automation watcher is enabled. Start from a fresh config (all watchers off) and use a scratch @@ -1785,7 +1843,7 @@ Expected result: seconds before the helper exits) still ends with a helper running: enable waits for an exiting helper to finish, then starts a new one. -## 12. Context-Size Estimate for a GGUF Model +## 13. Context-Size Estimate for a GGUF Model Needs a host with a Radeon GPU (a discrete card: an APU answers `undetermined` by design) and network access to Hugging Face. The point is to check the @@ -1848,7 +1906,7 @@ Finally, `rocm diagnose --report --model /:Q4_K_M --context-size 8192` prints the report with no model section, and stderr says that `--model, --context-size were ignored` and nothing about the model was fetched. -## 13. Pasting An Error Into `rocm diagnose` +## 14. Pasting An Error Into `rocm diagnose` `rocm diagnose --symptom-file` takes a pasted error without shell quoting. Copy a line that would break an argument, for example: @@ -1886,7 +1944,7 @@ Expected result: paste with a non-ASCII character in it (`é`, `→`, a typographic quote) reports the same causes as in PowerShell 7. -## 14. Which Causes a Pasted Error Contributed To +## 15. Which Causes a Pasted Error Contributed To `rocm diagnose --symptom` scores the catalog from the examined host's state and the pasted text together, so a finding about the host can outrank the entry the @@ -1914,7 +1972,7 @@ HSA_OVERRIDE_GFX_VERSION=11.0.0 rocm diagnose --json The e2e scenarios `diagnose-62`, `diagnose-63` and `diagnose-64` run these three checks on every lane. -## 15. Dashboard Chat With A Small-Context Local Model +## 16. Dashboard Chat With A Small-Context Local Model Needs a local OpenAI-compatible endpoint serving a model with a small context — llama.cpp's default `n_ctx` is 32768, which is the case this checks. The @@ -1956,7 +2014,7 @@ Expected result: context.` — and `rocm-cli.log` should still carry the provider's own body with `exceed_context_size_error` in it. -## 16. Serve Smoke Test Failure +## 17. Serve Smoke Test Failure Needs any engine `rocm serve` can start (the behaviour is not vLLM-specific) and an interactive terminal: the smoke test and its summary run only there. The diff --git a/docs/testing.md b/docs/testing.md index 9c42bb5fd..cf1b13636 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1658,18 +1658,19 @@ self-hosted CI dispatch (which forwards `--name`): each bypasses every gate above, so on WSL2, Windows or a machine with no GPU it would still run the real install. -`serve-26` and `serve-27` (`features/model_serving.feature`, the AIM serve and -restart canaries) are `@nightly` for cost: each pulls the AIM image and a 9B +`serve-26`, `serve-27` and `serve-62` (`features/model_serving.feature`, the +AIM serve and restart canaries and the AIM endpoint-key check) are `@nightly` +for cost: each pulls the AIM image and a 9B checkpoint on a cold runner and may wait up to 1200s for readiness, which on the Strix Ubuntu job, back when it ran on pull requests (about 25 minutes without them, 45 minute cap) used up the whole budget. They still need `@requires-engine:aim`, i.e. Docker reachable and a Radeon or Strix Halo GPU, so they run only where `E2E_INCLUDE_NIGHTLY=1` is set on such a host: the nightly Strix Halo lane, or an `include_nightly` dispatch of `e2e-selfhosted.yml` (whose Strix Ubuntu job then -gets a 105 minute cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins +gets a 125 minute cap). `aim_serve_scenarios_stay_off_the_per_pr_lanes` pins the tagging and that they stay reachable when the opt-in is set. -Both scenarios skip on a host without Docker; the `e2e-gpu-nightly-strix` job +All three scenarios skip on a host without Docker; the `e2e-gpu-nightly-strix` job logs whether Docker is reachable ("Report whether Docker is reachable"). `serve-27` failed to confirm the AIM stop on the Strix Halo pool (rocm-cli#1058) and passed on the 2026-11-15 and 2026-11-16 nightlies, so it runs expect-pass @@ -2363,13 +2364,17 @@ audit log. Only the serving user can read the key file: it is mode 0600 on Unix and has a protected DACL granting only that user on Windows, and every store replaces it with a freshly created owner-only file, so permissions loosened in place do not survive a re-store. Verifying that the running server actually -*rejects* an unauthenticated request requires a live engine; that end-to-end -assertion is a deferred follow-up and is **not yet** exercised by the GPU -acceptance scripts (`scripts/vllm_therock_gpu_test.py`). The unit tests cover -key policy, generation, the owner-only key file (including one whose -permissions were loosened before a re-store, and on the Windows build job the -DACL read back from disk), and key-file resolution. The tests that fail -without the repair of a loosened key file are `restoring_a_key_over_a_loosened_file_makes_it_0600_again` +*rejects* an unauthenticated request requires a live engine. For vLLM that is +the `@nightly` GPU scenario `serve-vllm-key-guards-every-route` (serve-61), +which runs in the nightly runs on Instinct GPUs (see "vLLM endpoint-key +coverage" below). For AIM it is the `@nightly` scenario +`serve-aim-key-guards-every-route` (serve-62; see "AIM endpoint-key coverage" +below), and `rocm serve` with AIM also checks it itself before serving. For +the other engines no test yet asserts that the running server refuses a keyless +request. The unit tests cover key policy, generation, the owner-only key file +(including one whose permissions were loosened before a re-store, and on the +Windows build job the DACL read back from disk), and key-file resolution. The +tests that fail without the repair of a loosened key file are `restoring_a_key_over_a_loosened_file_makes_it_0600_again` in [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs), `write_owner_only_file_atomically_tightens_a_loosened_file` in [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs), and on Windows `write_owner_only_file_atomically_replaces_a_loose_dacl_with_an_owner_only_one` @@ -2503,8 +2508,10 @@ via `spawn_hidden_console_with_log`, whose env-override API is path-valued only, so it cannot receive the value-typed `LEMONADE_API_KEY` that Lemonade's server reads. Rather than launch an unauthenticated public server, `rocm serve` **fails closed** on that exact combination (public bind + `--engine lemonade` + Windows) -and directs the user to `--engine vllm` or a loopback host. vLLM (all platforms, -via `VLLM_API_KEY`), direct llama-server serving (via `--api-key-file`), +and directs the user to `--engine vllm` or a loopback host. vLLM (Linux and WSL) +and AIM, both via `VLLM_API_KEY` plus the `rocm_cli_vllm_auth` middleware +described below, +direct llama-server serving (via `--api-key-file`), and Lemonade on non-Windows are all fully supported. Teaching the Windows detached-spawn primitive to carry a value-typed override (so Windows Lemonade can serve public with auth) is tracked as a separate follow-up. @@ -2529,6 +2536,89 @@ lemonade` against the shared tree) reaches the restore only on a runner whose shared tree still holds a stale pin; on a healthy runner the install succeeds and the restore never runs, so a green lane does not show that it works. +vLLM endpoint-key coverage: `VLLM_API_KEY` on its own makes vLLM check the key +only under a fixed set of path prefixes (`/v1` and a few others), while the +server also answers routes such as `/invocations`, `/tokenize` and `/metrics`. +So a keyed vLLM launch also passes `--middleware +rocm_cli_vllm_auth.RequireEndpointApiKey`, an ASGI middleware compiled into the +binary +([rocm_cli_vllm_auth.py](../crates/rocm-engine-protocol/src/rocm_cli_vllm_auth.py), +shared by both engines that run vLLM through +[endpoint_auth.rs](../crates/rocm-engine-protocol/src/endpoint_auth.rs)), written +under the engine's data directory at launch and put on the server's +`PYTHONPATH`. It +requires the key on every route except `GET`/`HEAD /health` and CORS +preflights, and raises instead of constructing when the key is empty, so a +server that loaded it never answers a request unchecked. + +```bash +cargo test -p rocm-engine-protocol endpoint_auth +cargo test -p rocm-engine-vllm endpoint_auth +cargo test -p rocm-engine-vllm vllm_serve_args +``` + +The `rocm-engine-protocol` `endpoint_auth` tests run the shipped middleware +under `python3` with a standard-library ASGI harness (they fail, rather than +skip, when `python3` is missing on Linux), so the per-route behavior is covered +on every PR without a GPU. The `vllm_server_command` tests assert, on every PR, that a keyed launch +gets the `--middleware` argument, `VLLM_API_KEY`, and the middleware directory +at the front of `PYTHONPATH` ahead of the runtime's own entry, together, and +that an unkeyed launch gets none of them: + +```bash +cargo test -p rocm-engine-vllm vllm_server_command +``` + +What no unit test covers is the call in `spawn_vllm_server` that hands the +resolved endpoint key to `vllm_server_command`: that spawn path probes the host +(OpenMPI, libatomic, libnuma) before it builds the command, so only a real +launch exercises it. + +That a real vLLM server loads the middleware and refuses keyless calls is +covered by the `@nightly` GPU scenario `serve-vllm-key-guards-every-route` +([model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature)), +which runs in the scheduled nightly runs on Instinct GPUs, where vLLM is the +serving engine: the MI300X job in [nightly.yml](../.github/workflows/nightly.yml) +and the shared GPU queue's Instinct lanes, MI350P among them +([nightly-on-the-queue.md](nightly-on-the-queue.md)). Those run after merge, not +per PR. It is not tagged `@merge-queue`: +the merge queue runs no lane where vLLM is the engine (the MI300X lane does not +run on `merge_group`), so under that tag it would run nowhere. To run it on a +branch before merge, dispatch `e2e-selfhosted.yml` on the MI300X lane with a +`name_filter`, which bypasses the harness's tag gates (`@nightly` included), so +the scenario runs without `include_nightly`: + +```bash +gh workflow run e2e-selfhosted.yml --ref \ + -f platform=app-dev-gpu -f name_filter='^serve-61 ' +``` + +AIM endpoint-key coverage: an AIM container bundles vLLM, so the same gap +applies to it. A keyed AIM launch +([endpoint_auth.rs](../engines/aim/src/endpoint_auth.rs)) bind-mounts the +middleware's directory read-only at `/opt/rocm-cli/python`, prepends that to +the container's `PYTHONPATH` (keeping the caller's or the image's value) and +adds `"middleware"` to `AIM_ENGINE_ARGS` (keeping any arguments already there), +all through the container's env file. Since the container's vLLM is not under +rocm-cli's control, `serve_http` then calls `POST /tokenize` without the key +once the container is ready, and stops the container and fails unless that is +refused with `401`. + +```bash +cargo test -p rocm-engine-aim endpoint_auth +cargo test -p rocm-engine-aim keyed +cargo test -p rocm-engine-aim run_args_mount +``` + +These cover the env and mount built for a keyed launch against one without a +key (including an image whose own `PYTHONPATH` and `AIM_ENGINE_ARGS` must +survive), the launch refused when the image's config cannot be read or its +`AIM_ENGINE_ARGS` cannot carry the middleware, and the refusal against a fake +server (`spawn_fake_server`) that does or does not check every route. That a +real AIM image applies `AIM_ENGINE_ARGS` and loads the middleware is covered +only by the `@nightly` scenario `serve-aim-key-guards-every-route`, which has +the same Docker caveat as `serve-26` and `serve-27` above. + Engine recipe adapter contract focused tests: ```bash @@ -4981,9 +5071,9 @@ hold the behaviour end to end, GPU or not: runs only the bash half, on Linux, and ci.yml's Windows lane does not run this scenario. - `diagnose-60` (Linux and WSL lanes only, like every pseudo-terminal - scenario here; the Windows key is checked by hand in - [manual-testing.md](manual-testing.md) section 13) runs - `rocm diagnose --symptom-file -` on a pseudo-terminal tall enough to keep + scenario here; the Windows key is checked by hand in the "Pasting An Error + Into `rocm diagnose`" section of [manual-testing.md](manual-testing.md)) + runs `rocm diagnose --symptom-file -` on a pseudo-terminal tall enough to keep the whole exchange on screen (it has no scrollback), requires the hint telling the user how to end the paste, then types the paste and Ctrl-D, the Linux end-of-input key the hint names, and requires the screen, which also diff --git a/engines/aim/src/docker.rs b/engines/aim/src/docker.rs index 94e95d6ee..6a409f229 100644 --- a/engines/aim/src/docker.rs +++ b/engines/aim/src/docker.rs @@ -56,6 +56,38 @@ pub(crate) struct RunSpec<'a> { /// `KEY=VALUE` entries go through a 0600 `--env-file`, never argv. An empty /// value means "forward `KEY` from this process's environment" (`-e KEY`). pub env: &'a BTreeMap, + /// Host directories bind-mounted read-only into the container. + pub read_only_mounts: &'a [ReadOnlyMount<'a>], +} + +/// A host directory (`source`) mounted read-only at `target` in the container. +pub(crate) struct ReadOnlyMount<'a> { + pub source: &'a Path, + pub target: &'a str, +} + +/// `--mount` arguments for `mounts`. `--mount` rather than `-v`, whose +/// `source:target` form is ambiguous for a Windows source path with a drive +/// letter. Its value is a comma-separated list, so a path containing a comma +/// or a quote is refused rather than split into a different mount. +fn mount_args(mounts: &[ReadOnlyMount<'_>]) -> Result> { + let mut args = Vec::new(); + for mount in mounts { + let source = mount.source.display().to_string(); + for part in [source.as_str(), mount.target] { + if part.contains([',', '"']) { + bail!( + "cannot bind-mount `{part}` into the container: the path contains `,` or `\"`" + ); + } + } + args.push("--mount".to_owned()); + args.push(format!( + "type=bind,source={source},target={},readonly", + mount.target + )); + } + Ok(args) } /// Splits `env` into entries destined for the `--env-file` and bare names to @@ -89,7 +121,7 @@ fn run_args( wsl: bool, env_file: Option<&Path>, forwarded: &[&str], -) -> Vec { +) -> Result> { let mut args = vec![ "run".to_owned(), "-d".to_owned(), @@ -118,8 +150,9 @@ fn run_args( args.push("-e".to_owned()); args.push((*key).to_owned()); } + args.extend(mount_args(spec.read_only_mounts)?); args.push(spec.image.to_owned()); - args + Ok(args) } fn write_env_file(entries: &[(&str, &str)]) -> Result { @@ -155,7 +188,7 @@ pub(crate) fn run(spec: &RunSpec<'_>) -> Result<()> { rocm_core::is_wsl_host(), env_file.as_ref().map(tempfile::NamedTempFile::path), &forwarded, - ); + )?; let mut command = Command::new(DOCKER_BIN); command.args(&args).stdin(Stdio::null()); @@ -401,8 +434,8 @@ pub(crate) fn daemon_reachable() -> bool { #[cfg(test)] mod tests { use super::{ - ContainerState, RunSpec, container_name, gpu_device_args, is_no_such, is_nothing_to_stop, - parse_container_state, parse_getent_gid, run_args, split_env, + ContainerState, ReadOnlyMount, RunSpec, container_name, gpu_device_args, is_no_such, + is_nothing_to_stop, parse_container_state, parse_getent_gid, run_args, split_env, }; use std::collections::BTreeMap; use std::path::Path; @@ -428,6 +461,7 @@ mod tests { use_gpu: !gpu_indices.is_empty(), render_gid: Some(993), env, + read_only_mounts: &[], } } @@ -441,7 +475,7 @@ mod tests { ] { let mut s = spec(&[], &env); s.host = host; - let args = run_args(&s, false, None, &[]); + let args = run_args(&s, false, None, &[]).unwrap(); assert_eq!(args[4], "-p"); assert_eq!(args[5], published); } @@ -485,7 +519,7 @@ mod tests { fn run_args_without_gpu_or_env_is_just_name_port_and_image() { let env = BTreeMap::new(); assert_eq!( - run_args(&spec(&[], &env), false, None, &[]), + run_args(&spec(&[], &env), false, None, &[]).unwrap(), vec![ "run", "-d", @@ -501,7 +535,7 @@ mod tests { #[test] fn run_args_with_gpu_sets_visible_devices_and_device_flags_before_the_image() { let env = BTreeMap::new(); - let args = run_args(&spec(&[1, 3], &env), false, None, &[]); + let args = run_args(&spec(&[1, 3], &env), false, None, &[]).unwrap(); assert_eq!(args.last().map(String::as_str), Some("org/aim-foo:1")); assert!( args.windows(2) @@ -516,7 +550,7 @@ mod tests { let env = BTreeMap::new(); let mut gpu_spec = spec(&[], &env); gpu_spec.use_gpu = true; - let args = run_args(&gpu_spec, false, None, &[]); + let args = run_args(&gpu_spec, false, None, &[]).unwrap(); assert!( !args .iter() @@ -540,7 +574,8 @@ mod tests { false, Some(Path::new("/tmp/env-file")), &forwarded, - ); + ) + .unwrap(); assert!( args.windows(2) .any(|pair| pair == ["--env-file", "/tmp/env-file"]) @@ -550,6 +585,69 @@ mod tests { assert_eq!(args.last().map(String::as_str), Some("org/aim-foo:1")); } + #[test] + fn run_args_mount_each_directory_read_only_before_the_image() { + let env = BTreeMap::new(); + let mounts = [ReadOnlyMount { + source: Path::new("/data/engines/aim/python"), + target: "/opt/rocm-cli/python", + }]; + let mut mounted = spec(&[], &env); + mounted.read_only_mounts = &mounts; + let args = run_args(&mounted, false, None, &[]).unwrap(); + let mount = args + .windows(2) + .position(|pair| { + pair == [ + "--mount", + "type=bind,source=/data/engines/aim/python,target=/opt/rocm-cli/python,readonly", + ] + }) + .expect("a read-only bind mount"); + assert!( + mount < args.len() - 2, + "mount comes before the image: {args:?}" + ); + assert_eq!(args.last().map(String::as_str), Some("org/aim-foo:1")); + } + + #[test] + fn run_args_refuse_a_mount_path_docker_would_split() { + let env = BTreeMap::new(); + // `,` separates `--mount` fields and `"` starts a quoted one, so either + // character in either path would let it say something else. + for splitter in [',', '"'] { + let unsafe_source = format!("/data/a{splitter}target=/etc"); + let unsafe_target = format!("/opt/rocm-cli/python{splitter}readonly=false"); + let cases = [ + ( + unsafe_source.as_str(), + "/opt/rocm-cli/python", + &unsafe_source, + ), + ( + "/data/engines/aim/python", + unsafe_target.as_str(), + &unsafe_target, + ), + ]; + for (source, target, offending) in cases { + let mounts = [ReadOnlyMount { + source: Path::new(source), + target, + }]; + let mut mounted = spec(&[], &env); + mounted.read_only_mounts = &mounts; + let error = run_args(&mounted, false, None, &[]) + .expect_err("a path that splits --mount must be refused"); + assert!( + error.to_string().contains(offending.as_str()), + "the refusal must name `{offending}`: {error}" + ); + } + } + } + #[test] fn parse_container_state_reads_running_and_exit_code() { assert_eq!( @@ -769,6 +867,7 @@ exit 0 gpu_indices: &[], render_gid: None, env, + read_only_mounts: &[], } } diff --git a/engines/aim/src/endpoint_auth.rs b/engines/aim/src/endpoint_auth.rs new file mode 100644 index 000000000..ce672d394 --- /dev/null +++ b/engines/aim/src/endpoint_auth.rs @@ -0,0 +1,430 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Endpoint API-key enforcement for an AIM container. +//! +//! AIM images bundle vLLM's OpenAI server, so an endpoint key reaches it as +//! `VLLM_API_KEY` — and, as with the native vLLM engine, that check alone +//! guards only a fixed set of path prefixes (`/v1` and a few others) while the +//! server also answers `/invocations`, `/tokenize`, `/metrics` and more. A +//! keyed launch therefore also installs the shared every-route middleware +//! (`rocm_engine_protocol::endpoint_auth`): +//! +//! - the module is written under this engine's data directory and bind-mounted +//! read-only into the container at [`CONTAINER_MODULE_DIR`]; +//! - that directory is prepended to the container's `PYTHONPATH`, ahead of the +//! value the caller or the image would otherwise have given it; +//! - `--middleware` is added through `AIM_ENGINE_ARGS`, the JSON object of +//! extra `vllm serve` arguments the AIM runtime merges into the selected +//! profile's own, keeping any arguments the caller passed there. +//! +//! All three travel in the container's env file, never on the `docker` command +//! line. Because the container's vLLM is not under this process's control, +//! the supervisor does not take the middleware's presence on trust: once the +//! container is ready it calls [`PROBE_ROUTE`] without the key and refuses to +//! serve unless that is refused ([`verify_every_route_guarded`]). + +use anyhow::{Context, Result, bail}; +use rocm_engine_protocol::endpoint_auth::{ + MIDDLEWARE_IMPORT_PATH, MIDDLEWARE_KEY_ENV, write_middleware_module, +}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +/// Where the middleware directory is mounted inside the container. +pub(crate) const CONTAINER_MODULE_DIR: &str = "/opt/rocm-cli/python"; + +/// The AIM runtime's variable for extra engine arguments: a JSON object whose +/// keys are `vllm serve` flag names without the leading `--`. +pub(crate) const ENGINE_ARGS_ENV: &str = "AIM_ENGINE_ARGS"; + +const PYTHONPATH_ENV: &str = "PYTHONPATH"; + +/// The `AIM_ENGINE_ARGS` key that becomes vLLM's `--middleware`. +const MIDDLEWARE_ENGINE_ARG: &str = "middleware"; + +/// A route vLLM's own key check leaves open, called without the key once the +/// container is ready. vLLM's own check lets it through, so a refusal shows the +/// middleware is installed; it samples one route rather than proving every one. +pub(crate) const PROBE_ROUTE: &str = "/tokenize"; + +const PROBE_TIMEOUT: Duration = Duration::from_secs(10); + +/// Host directory the middleware module is written to, under this engine's +/// data directory. +pub(crate) fn module_dir(paths: &rocm_core::AppPaths) -> PathBuf { + paths.engine_dir(crate::ENGINE_NAME).join("python") +} + +/// Write the middleware into `module_dir` and return that directory, the +/// source of the container's read-only bind mount. +pub(crate) fn stage_module(module_dir: &Path) -> Result { + write_middleware_module(module_dir)?; + Ok(module_dir.to_path_buf()) +} + +/// Add the endpoint key and the middleware to a keyed container's `env`. +/// +/// Sets `VLLM_API_KEY`, prepends [`CONTAINER_MODULE_DIR`] to `PYTHONPATH` and +/// adds `middleware` to `AIM_ENGINE_ARGS`. The values those two would +/// otherwise have had are kept: the caller's (`--engine-env`, where an empty +/// value means "forward it from this process's environment", resolved through +/// `host_env`), else the image's own (`image_env`, from its config) — setting +/// either in the env file replaces the image's value, so dropping it would +/// silently change what the server imports or how it runs. +/// +/// Fails rather than launch when the `AIM_ENGINE_ARGS` it would extend — the +/// caller's, one forwarded from this process's environment, or the image's — +/// is not a JSON object or already names a middleware: neither can be +/// combined with the one the key needs. The refusal names the fix that suits +/// where the value came from ([`ValueSource`]). +pub(crate) fn protect_container_env( + env: &mut BTreeMap, + api_key: &str, + image_env: &BTreeMap, + host_env: impl Fn(&str) -> Option, +) -> Result<()> { + let pythonpath = match inherited_value(env, image_env, PYTHONPATH_ENV, &host_env)? { + Some(rest) if !rest.is_empty() => format!("{CONTAINER_MODULE_DIR}:{rest}"), + _ => CONTAINER_MODULE_DIR.to_owned(), + }; + let engine_args_source = match env.get(ENGINE_ARGS_ENV) { + Some(value) if value.is_empty() => ValueSource::Host, + Some(_) => ValueSource::Caller, + None => ValueSource::Image, + }; + let engine_args = engine_args_with_middleware( + inherited_value(env, image_env, ENGINE_ARGS_ENV, &host_env)?.as_deref(), + engine_args_source, + )?; + env.insert(MIDDLEWARE_KEY_ENV.to_owned(), api_key.to_owned()); + env.insert(PYTHONPATH_ENV.to_owned(), pythonpath); + env.insert(ENGINE_ARGS_ENV.to_owned(), engine_args); + Ok(()) +} + +/// The value `key` would have in the container without this module: the +/// caller's, else the image's. +fn inherited_value( + env: &BTreeMap, + image_env: &BTreeMap, + key: &str, + host_env: &impl Fn(&str) -> Option, +) -> Result> { + match env.get(key) { + Some(value) if value.is_empty() => host_env(key).map(Some).with_context(|| { + format!( + "engine env `{key}` was passed without a value, but `{key}` is not set in this \ + process's environment; export it before serving or pass `{key}=VALUE`" + ) + }), + Some(value) => Ok(Some(value.clone())), + None => Ok(image_env.get(key).cloned()), + } +} + +/// Where an inherited `AIM_ENGINE_ARGS` came from, which decides how a user +/// gets rid of one that cannot carry the middleware. +#[derive(Debug, Clone, Copy)] +enum ValueSource { + /// Set with a value by this launch (`--engine-env KEY=VALUE` or the + /// engine recipe): the user changes it where they set it. + Caller, + /// Named with no value, so forwarded from this process's environment: + /// the bad value lives there, not on the command line. + Host, + /// The image's own config: the user cannot remove it, only override it. + Image, +} + +impl ValueSource { + fn remedy(self) -> String { + match self { + Self::Caller => format!( + "change `{ENGINE_ARGS_ENV}` where this launch sets it (`--engine-env` or the \ + engine recipe), or serve without an API key on a loopback host" + ), + Self::Host => format!( + "`--engine-env {ENGINE_ARGS_ENV}` with no value forwards it from this process's \ + environment, so set it there to a JSON object naming no \ + `{MIDDLEWARE_ENGINE_ARG}`, or replace that flag with `--engine-env \ + {ENGINE_ARGS_ENV}=`, or serve without an API key on a loopback host" + ), + Self::Image => format!( + "the image sets this value, so override it with `--engine-env \ + {ENGINE_ARGS_ENV}=` naming no `{MIDDLEWARE_ENGINE_ARG}` (it replaces \ + the image's value whole, so repeat any of its other arguments you still want), \ + or serve without an API key on a loopback host" + ), + } + } +} + +/// `existing` (a JSON object, or nothing) with `middleware` set to the +/// endpoint-key middleware, serialized for `AIM_ENGINE_ARGS`. `source` says +/// where `existing` came from, for the remedy a refusal names. +fn engine_args_with_middleware(existing: Option<&str>, source: ValueSource) -> Result { + let mut args = match existing.map(str::trim).filter(|value| !value.is_empty()) { + None => serde_json::Map::new(), + Some(text) => match serde_json::from_str::(text) { + Ok(serde_json::Value::Object(map)) => map, + _ => bail!( + "`{ENGINE_ARGS_ENV}` must be a JSON object to be combined with the endpoint \ + API key's middleware, got `{text}`; {}", + source.remedy() + ), + }, + }; + if args.contains_key(MIDDLEWARE_ENGINE_ARG) { + bail!( + "`{ENGINE_ARGS_ENV}` already sets `{MIDDLEWARE_ENGINE_ARG}`, which rocm-cli needs \ + to enforce the endpoint API key on every route; {}", + source.remedy() + ); + } + args.insert( + MIDDLEWARE_ENGINE_ARG.to_owned(), + serde_json::Value::String(MIDDLEWARE_IMPORT_PATH.to_owned()), + ); + Ok(serde_json::Value::Object(args).to_string()) +} + +/// Confirm a ready keyed container refuses a keyless call to [`PROBE_ROUTE`]. +/// +/// The middleware answers `401` before routing; a server without it answers +/// the request itself. Anything but `401`, or no answer, is an error, so a +/// container whose image ignored the middleware is not left serving with +/// [`PROBE_ROUTE`] open. +pub(crate) fn verify_every_route_guarded(container: &str, endpoint_url: &str) -> Result<()> { + let response = rocm_core::http_post_json_with_auth( + endpoint_url, + PROBE_ROUTE, + &serde_json::json!({}), + None, + PROBE_TIMEOUT, + ) + .with_context(|| { + format!( + "could not confirm that AIM container `{container}` requires the endpoint API key \ + on every route" + ) + })?; + route_guard_verdict(container, response.status) +} + +fn route_guard_verdict(container: &str, status: u16) -> Result<()> { + if status == 401 { + return Ok(()); + } + bail!( + "AIM container `{container}` answered `POST {PROBE_ROUTE}` without the endpoint API key \ + (HTTP {status}), so the key would not protect every route; refusing to serve it. The \ + image may not apply `{ENGINE_ARGS_ENV}` or vLLM's `--middleware`." + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn no_host_env(_: &str) -> Option { + None + } + + fn engine_args(env: &BTreeMap) -> serde_json::Value { + serde_json::from_str(&env[ENGINE_ARGS_ENV]).expect("AIM_ENGINE_ARGS is JSON") + } + + #[test] + fn a_keyed_container_gets_the_key_the_module_path_and_the_middleware() -> Result<()> { + let mut env = BTreeMap::from([("AIM_PROFILE_ID".to_owned(), "p".to_owned())]); + protect_container_env(&mut env, "k3y", &BTreeMap::new(), no_host_env)?; + assert_eq!(env["VLLM_API_KEY"], "k3y"); + assert_eq!(env["PYTHONPATH"], CONTAINER_MODULE_DIR); + assert_eq!( + engine_args(&env), + serde_json::json!({"middleware": MIDDLEWARE_IMPORT_PATH}) + ); + assert_eq!(env["AIM_PROFILE_ID"], "p", "other entries are untouched"); + Ok(()) + } + + #[test] + fn the_images_pythonpath_and_engine_args_are_kept_behind_the_middleware() -> Result<()> { + let image_env = BTreeMap::from([ + ("PYTHONPATH".to_owned(), "/workspace/aim-runtime".to_owned()), + ( + ENGINE_ARGS_ENV.to_owned(), + r#"{"max-model-len": 4096}"#.to_owned(), + ), + ]); + let mut env = BTreeMap::new(); + protect_container_env(&mut env, "k3y", &image_env, no_host_env)?; + assert_eq!( + env["PYTHONPATH"], + format!("{CONTAINER_MODULE_DIR}:/workspace/aim-runtime") + ); + assert_eq!( + engine_args(&env), + serde_json::json!({"max-model-len": 4096, "middleware": MIDDLEWARE_IMPORT_PATH}) + ); + Ok(()) + } + + #[test] + fn the_callers_values_win_over_the_images_and_are_kept() -> Result<()> { + let image_env = BTreeMap::from([("PYTHONPATH".to_owned(), "/image".to_owned())]); + let mut env = BTreeMap::from([ + ("PYTHONPATH".to_owned(), "/caller".to_owned()), + ( + ENGINE_ARGS_ENV.to_owned(), + r#"{"enforce-eager": null}"#.to_owned(), + ), + ]); + protect_container_env(&mut env, "k3y", &image_env, no_host_env)?; + assert_eq!(env["PYTHONPATH"], format!("{CONTAINER_MODULE_DIR}:/caller")); + assert_eq!( + engine_args(&env), + serde_json::json!({"enforce-eager": null, "middleware": MIDDLEWARE_IMPORT_PATH}) + ); + Ok(()) + } + + #[test] + fn a_forwarded_caller_value_is_read_from_this_process() -> Result<()> { + let mut env = BTreeMap::from([(ENGINE_ARGS_ENV.to_owned(), String::new())]); + protect_container_env(&mut env, "k3y", &BTreeMap::new(), |key| { + (key == ENGINE_ARGS_ENV).then(|| r#"{"seed": 7}"#.to_owned()) + })?; + assert_eq!( + engine_args(&env), + serde_json::json!({"seed": 7, "middleware": MIDDLEWARE_IMPORT_PATH}) + ); + + let mut env = BTreeMap::from([(ENGINE_ARGS_ENV.to_owned(), String::new())]); + let error = protect_container_env(&mut env, "k3y", &BTreeMap::new(), no_host_env) + .expect_err("a forwarded variable that is unset cannot be merged"); + assert!(error.to_string().contains("not set"), "{error}"); + Ok(()) + } + + #[test] + fn engine_args_that_cannot_carry_the_middleware_are_refused() { + for existing in [ + r#"{"middleware": "other.Middleware"}"#, + r#"["--seed", "7"]"#, + "not json", + ] { + let mut env = BTreeMap::from([(ENGINE_ARGS_ENV.to_owned(), existing.to_owned())]); + let error = protect_container_env(&mut env, "k3y", &BTreeMap::new(), no_host_env) + .expect_err(existing); + let text = error.to_string(); + assert!(text.contains(ENGINE_ARGS_ENV), "{text}"); + assert!( + text.contains("change `AIM_ENGINE_ARGS` where this launch sets it"), + "a caller's value is the caller's to change: {text}" + ); + assert!( + !env.contains_key("VLLM_API_KEY"), + "nothing is set when the launch is refused" + ); + } + } + + #[test] + fn an_images_engine_args_that_cannot_carry_the_middleware_name_the_override() -> Result<()> { + for existing in [r#"{"middleware": "other.Middleware"}"#, "not json"] { + let image_env = BTreeMap::from([(ENGINE_ARGS_ENV.to_owned(), existing.to_owned())]); + let mut env = BTreeMap::new(); + let error = protect_container_env(&mut env, "k3y", &image_env, no_host_env) + .expect_err(existing); + let text = error.to_string(); + assert!( + text.contains("the image sets this value") + && text.contains("--engine-env AIM_ENGINE_ARGS=") + && text.contains("replaces the image's value whole"), + "an image's value can only be overridden, and the refusal must say how: {text}" + ); + assert!(env.is_empty(), "nothing is set when the launch is refused"); + + // The override the refusal names does clear it, replacing the + // image's value rather than merging with it. + let mut env = + BTreeMap::from([(ENGINE_ARGS_ENV.to_owned(), r#"{"seed": 7}"#.to_owned())]); + protect_container_env(&mut env, "k3y", &image_env, no_host_env)?; + assert_eq!( + engine_args(&env), + serde_json::json!({"seed": 7, "middleware": MIDDLEWARE_IMPORT_PATH}) + ); + } + Ok(()) + } + + #[test] + fn a_forwarded_engine_args_that_cannot_carry_the_middleware_points_at_the_environment() + -> Result<()> { + let mut env = BTreeMap::from([(ENGINE_ARGS_ENV.to_owned(), String::new())]); + let host = |key: &str| (key == ENGINE_ARGS_ENV).then(|| "not json".to_owned()); + let error = protect_container_env(&mut env, "k3y", &BTreeMap::new(), host) + .expect_err("a forwarded value that is not a JSON object is refused"); + let text = error.to_string(); + assert!( + text.contains("forwards it from this process's environment") + && text.contains("set it there to a JSON object") + && text.contains( + "replace that flag with `--engine-env AIM_ENGINE_ARGS=`" + ), + "the refusal must name both steps that clear it: {text}" + ); + assert!( + !env.contains_key("VLLM_API_KEY"), + "nothing is set when the launch is refused" + ); + + // Each step it names clears the refusal. Setting the forwarded value to + // a JSON object, keeping the bare flag: + let fixed_host = |key: &str| (key == ENGINE_ARGS_ENV).then(|| r#"{"seed": 7}"#.to_owned()); + let mut env = BTreeMap::from([(ENGINE_ARGS_ENV.to_owned(), String::new())]); + protect_container_env(&mut env, "k3y", &BTreeMap::new(), fixed_host)?; + assert_eq!( + engine_args(&env), + serde_json::json!({"seed": 7, "middleware": MIDDLEWARE_IMPORT_PATH}) + ); + // Replacing the flag with an explicit value, the environment unchanged: + let mut env = BTreeMap::from([(ENGINE_ARGS_ENV.to_owned(), r#"{"seed": 7}"#.to_owned())]); + protect_container_env(&mut env, "k3y", &BTreeMap::new(), host)?; + assert_eq!( + engine_args(&env), + serde_json::json!({"seed": 7, "middleware": MIDDLEWARE_IMPORT_PATH}) + ); + Ok(()) + } + + #[test] + fn only_a_refusal_on_the_probe_route_counts_as_guarded() { + assert!(route_guard_verdict("c", 401).is_ok()); + for status in [200, 400, 403, 404, 422, 500] { + let error = route_guard_verdict("c", status).expect_err("not guarded"); + let text = error.to_string(); + assert!(text.contains(&format!("HTTP {status}")), "{text}"); + assert!(text.contains("refusing to serve"), "{text}"); + } + } + + #[test] + fn stage_module_writes_the_middleware_into_the_mount_source() -> Result<()> { + let temp = tempfile::tempdir()?; + let dir = temp.path().join("python"); + assert_eq!(stage_module(&dir)?, dir); + assert!( + dir.join(format!( + "{}.py", + rocm_engine_protocol::endpoint_auth::MIDDLEWARE_MODULE_NAME + )) + .is_file() + ); + Ok(()) + } +} diff --git a/engines/aim/src/lib.rs b/engines/aim/src/lib.rs index 3bab19210..eb8f1e335 100644 --- a/engines/aim/src/lib.rs +++ b/engines/aim/src/lib.rs @@ -16,6 +16,7 @@ use std::io::{Read, Write}; use std::path::PathBuf; mod docker; +mod endpoint_auth; mod install; mod process; mod registry; diff --git a/engines/aim/src/process.rs b/engines/aim/src/process.rs index b6460269f..917ae196c 100644 --- a/engines/aim/src/process.rs +++ b/engines/aim/src/process.rs @@ -164,7 +164,10 @@ pub(crate) fn launch_service(request: LaunchRequest) -> Result { let state_path = crate::state::state_path(&request.service_id); - let served_model_id = run_container( + // Returns once `docker run -d` has created the container, before it is + // ready, so unlike `serve_http` (the path `rocm serve` takes) it cannot + // probe a keyed container's routes; the middleware is installed the same way. + let launched = run_container( &request.service_id, &request.model_ref, &request.host, @@ -178,7 +181,7 @@ pub(crate) fn launch_service(request: LaunchRequest) -> Result { &state_path, &request.service_id, &request.model_ref, - served_model_id.as_deref(), + launched.label_name.as_deref(), &request.host, request.port, device_policy, @@ -246,8 +249,17 @@ fn plan_gpus(requested: &[u32], usable: Option>, windows: bool) -> Resu }) } -/// Starts the named service's container unless it is already running, and -/// returns the model name the image says it serves under, if known. +/// What [`run_container`] started (or found already running). +#[derive(Debug)] +struct Launched { + /// The model name the image says it serves under, if known. + label_name: Option, + /// Whether the endpoint is protected by an API key, so the container must + /// refuse keyless calls on every route once ready. + keyed: bool, +} + +/// Starts the named service's container unless it is already running. /// /// Container names are unique in Docker, so `docker run --name X` against a /// name still held by an exited container (a crash, a reboot, a failed @@ -261,15 +273,20 @@ fn run_container( device_policy: &DevicePolicy, requested_gpus: &[u32], extra_env: Option<&BTreeMap>, -) -> Result> { +) -> Result { let name = crate::docker::container_name(service_id); + let api_key = rocm_engine_protocol::resolve_endpoint_api_key(); + let keyed = api_key.is_some(); // Normalize once so the pull, the metadata and the run all use the same image. let image_ref = crate::registry::normalize_image_ref(model_ref); match crate::docker::container_state(&name)? { ContainerState::Running => { - return Ok(crate::registry::local_image_metadata(&image_ref) - .and_then(|metadata| metadata.canonical_name)); + return Ok(Launched { + label_name: crate::registry::local_image_metadata(&image_ref) + .and_then(|metadata| metadata.canonical_name), + keyed, + }); } ContainerState::Exited(_) => crate::docker::remove(&name)?, ContainerState::Missing => {} @@ -280,7 +297,8 @@ fn run_container( } // Read from the pulled image itself, so this works for any registry and // offline, unlike the Docker Hub lookup. - let metadata = crate::registry::local_image_metadata(&image_ref).unwrap_or_default(); + let image_metadata = crate::registry::read_local_image_metadata(&image_ref); + let metadata = image_metadata.as_ref().cloned().unwrap_or_default(); let gpus = plan_gpus( requested_gpus, @@ -306,13 +324,41 @@ fn run_container( bail!("engine env `{key}` is managed by rocm-cli and cannot be overridden"); } // AIM images bundle vLLM's OpenAI server, so it is `VLLM_API_KEY` that this - // engine's actual server enforces. The container is the only process this - // engine runs, so without this the require-api-key setting a public - // `rocm serve --engine aim` was given is silently never enforced. The value - // reaches docker through a private env file, never argv. - if let Some(api_key) = rocm_engine_protocol::resolve_endpoint_api_key() { - env.insert("VLLM_API_KEY".to_owned(), api_key); - } + // engine's actual server enforces — but only under vLLM's own fixed path + // prefixes, so a keyed launch also mounts in the every-route middleware + // (`endpoint_auth`). The container is the only process this engine runs, so + // without this the require-api-key setting a public `rocm serve --engine + // aim` was given is never enforced. The values reach docker through a + // private env file, never argv. + let module_dir = match api_key.as_deref() { + Some(api_key) => { + // The env file replaces the image's own `PYTHONPATH` and + // `AIM_ENGINE_ARGS`, so a keyed launch has to know them to keep + // them; without the image's config it would drop them unseen. + if let Err(error) = &image_metadata { + bail!( + "cannot read the config of AIM image `{image_ref}`, which a key-protected \ + launch needs to keep the image's own `PYTHONPATH` and `AIM_ENGINE_ARGS`: \ + {error:#}" + ); + } + crate::endpoint_auth::protect_container_env(&mut env, api_key, &metadata.env, |key| { + std::env::var(key).ok() + })?; + Some(crate::endpoint_auth::stage_module( + &crate::endpoint_auth::module_dir(&rocm_core::AppPaths::discover()?), + )?) + } + None => None, + }; + let mounts: Vec> = module_dir + .as_deref() + .map(|source| crate::docker::ReadOnlyMount { + source, + target: crate::endpoint_auth::CONTAINER_MODULE_DIR, + }) + .into_iter() + .collect(); crate::docker::run(&crate::docker::RunSpec { name: &name, image: &image_ref, @@ -323,8 +369,12 @@ fn run_container( gpu_indices: if use_gpu { &gpus.indices } else { &[] }, render_gid, env: &env, + read_only_mounts: &mounts, })?; - Ok(metadata.canonical_name) + Ok(Launched { + label_name: metadata.canonical_name, + keyed, + }) } /// Enforces `device_policy` instead of treating GPU use as a soft heuristic: @@ -357,7 +407,7 @@ fn resolve_use_gpu( pub(crate) fn serve_http(request: ServeHttpRequest) -> Result<()> { let name = crate::docker::container_name(&request.service_id); - let label_name = run_container( + let launched = run_container( &request.service_id, &request.model_ref, &request.host, @@ -377,7 +427,7 @@ pub(crate) fn serve_http(request: ServeHttpRequest) -> Result<()> { &request.state_path, &request.service_id, &request.model_ref, - label_name.as_deref(), + launched.label_name.as_deref(), &request.host, request.port, request.device_policy, @@ -387,13 +437,26 @@ pub(crate) fn serve_http(request: ServeHttpRequest) -> Result<()> { // The container serves under the image's HF-style canonical name, not the // Docker image reference used for `docker pull`/`run`. When the image didn't // label one, wait for the container to list whatever it serves instead. - match wait_for_container_ready( + let ready = wait_for_container_ready( &name, &request.host, request.port, - label_name.as_deref(), + launched.label_name.as_deref(), aim_ready_timeout(), - ) { + ) + .and_then(|served_model_id| { + // A keyed container must refuse keyless calls on every route before + // it is reported as serving; otherwise it is torn down like any + // other failed start. + if launched.keyed { + crate::endpoint_auth::verify_every_route_guarded( + &name, + &crate::state::endpoint_url(&request.host, request.port), + )?; + } + Ok(served_model_id) + }); + match ready { Ok(served_model_id) => { crate::state::record_served_model_id(&request.state_path, &served_model_id)?; let exit = wait_for_exit( @@ -861,6 +924,14 @@ mod subprocess_tests { /// Serves `GET /v1/models` (listing `model`) and `POST /v1/chat/completions` /// (always 200) on an ephemeral loopback port; returns the port. fn spawn_fake_openai(model: &'static str) -> u16 { + spawn_fake_server(model, false) + } + + /// As [`spawn_fake_openai`], and when `guard_every_route` answers `401` to + /// any request without an `Authorization` header, whatever its path — as + /// the endpoint-key middleware does. Without it every other path answers + /// `200`, like a vLLM whose own key check skips that route. + fn spawn_fake_server(model: &'static str, guard_every_route: bool) -> u16 { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); std::thread::spawn(move || { @@ -870,28 +941,31 @@ mod subprocess_tests { let mut request_line = String::new(); reader.read_line(&mut request_line).ok(); let mut content_length = 0usize; + let mut authorized = false; loop { let mut header = String::new(); if reader.read_line(&mut header).unwrap_or(0) == 0 || header == "\r\n" { break; } - if let Some(value) = - header.to_ascii_lowercase().strip_prefix("content-length:") - { + let lower = header.to_ascii_lowercase(); + if let Some(value) = lower.strip_prefix("content-length:") { content_length = value.trim().parse().unwrap_or(0); } + authorized |= lower.starts_with("authorization:"); } let mut body = vec![0; content_length]; reader.read_exact(&mut body).ok(); - let payload = if request_line.starts_with("GET /v1/models") { - format!(r#"{{"data":[{{"id":"{model}"}}]}}"#) + let (status, payload) = if guard_every_route && !authorized { + ("401 Unauthorized", r#"{"error":"Unauthorized"}"#.to_owned()) + } else if request_line.starts_with("GET /v1/models") { + ("200 OK", format!(r#"{{"data":[{{"id":"{model}"}}]}}"#)) } else { - r#"{"choices":[]}"#.to_owned() + ("200 OK", r#"{"choices":[]}"#.to_owned()) }; let mut stream = stream; let _ = write!( stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{payload}", + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{payload}", payload.len() ); }); @@ -1264,14 +1338,28 @@ mod subprocess_tests { ], ); assert!(!fixture.argv().join("\n").contains("vllm-profile")); + assert!( + !fixture.argv().join("\n").contains("--mount"), + "an unkeyed launch mounts nothing: {:?}", + fixture.argv() + ); assert_eq!( fixture.env_copy().as_deref(), - Some("AIM_PROFILE_ID=vllm-profile\n") + Some("AIM_PROFILE_ID=vllm-profile\n"), + "an unkeyed launch adds no key, PYTHONPATH or AIM_ENGINE_ARGS" ); } + /// Writes an endpoint key file for a keyed launch and returns its path; + /// the caller removes it. + fn endpoint_key_file(tag: &str) -> std::path::PathBuf { + let key_file = std::env::temp_dir().join(format!("aim-key-{tag}-{}", std::process::id())); + std::fs::write(&key_file, "s3cret\n").unwrap(); + key_file + } + #[test] - fn run_container_forwards_the_endpoint_api_key_as_vllm_api_key() { + fn run_container_installs_the_every_route_middleware_for_a_keyed_launch() { if in_subprocess() { run_container( "svc-key", @@ -1283,15 +1371,167 @@ mod subprocess_tests { None, ) .unwrap(); + let module = + crate::endpoint_auth::module_dir(&rocm_core::AppPaths::discover().unwrap()) + .join("rocm_cli_vllm_auth.py"); + assert_eq!( + std::fs::read_to_string(&module).unwrap(), + rocm_engine_protocol::endpoint_auth::MIDDLEWARE_MODULE_SOURCE, + "the mounted directory must hold the middleware the container imports" + ); return; } - let key_file = std::env::temp_dir().join(format!("aim-key-{}", std::process::id())); - std::fs::write(&key_file, "s3cret\n").unwrap(); + let key_file = endpoint_key_file("run"); let fixture = run_with_fake_docker( - "process::subprocess_tests::run_container_forwards_the_endpoint_api_key_as_vllm_api_key", + "process::subprocess_tests::run_container_installs_the_every_route_middleware_for_a_keyed_launch", &[ ("FAKE_DOCKER_INSPECT_SEQUENCE", "missing"), ("FAKE_DOCKER_IMAGE_PRESENT_EXIT", "0"), + // The image's own values, which the env file replaces unless + // the launch carries them over. + ( + "FAKE_DOCKER_IMAGE_CONFIG", + r#"{"Env":["PYTHONPATH=/workspace/aim-runtime","AIM_ENGINE_ARGS={\"max-model-len\": 4096}"]}"#, + ), + ( + rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, + key_file.to_str().unwrap(), + ), + ], + ); + std::fs::remove_file(&key_file).ok(); + let argv = fixture.argv().join("\n"); + assert!( + !argv.contains("s3cret"), + "the key must stay off argv: {argv}" + ); + assert!( + argv.contains("--mount type=bind,source=") + && argv.contains("/engines/aim/python,target=/opt/rocm-cli/python,readonly"), + "the middleware directory must be mounted read-only: {argv}" + ); + assert_eq!( + fixture.env_copy().as_deref(), + Some( + "AIM_ENGINE_ARGS={\"max-model-len\":4096,\"middleware\":\"rocm_cli_vllm_auth.RequireEndpointApiKey\"}\n\ + PYTHONPATH=/opt/rocm-cli/python:/workspace/aim-runtime\n\ + VLLM_API_KEY=s3cret\n" + ), + "the image's PYTHONPATH and AIM_ENGINE_ARGS must survive behind the middleware" + ); + } + + #[test] + fn run_container_refuses_a_keyed_launch_whose_image_config_cannot_be_read() { + if in_subprocess() { + let error = run_container( + "svc-key-noconfig", + IMAGE, + "127.0.0.1", + 18090, + &DevicePolicy::CpuOnly, + &[], + None, + ) + .expect_err("a keyed launch must not guess the image's own env"); + let text = format!("{error:#}"); + assert!( + text.contains("cannot read the config of AIM image"), + "{text}" + ); + assert!( + text.contains("returned no readable `.Config`"), + "the refusal must carry why the config could not be read: {text}" + ); + return; + } + let key_file = endpoint_key_file("noconfig"); + // `docker image inspect` succeeds (the image is present) but yields no + // config: first nothing it can parse, then a `null` `.Config`. + for config in [None, Some("null")] { + let mut env = vec![ + ("FAKE_DOCKER_INSPECT_SEQUENCE", "missing"), + ("FAKE_DOCKER_IMAGE_PRESENT_EXIT", "0"), + ( + rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, + key_file.to_str().unwrap(), + ), + ]; + if let Some(config) = config { + env.push(("FAKE_DOCKER_IMAGE_CONFIG", config)); + } + let fixture = run_with_fake_docker( + "process::subprocess_tests::run_container_refuses_a_keyed_launch_whose_image_config_cannot_be_read", + &env, + ); + assert!( + !fixture.argv().iter().any(|line| line.starts_with("run ")), + "no container may start ({config:?}): {:?}", + fixture.argv() + ); + } + std::fs::remove_file(&key_file).ok(); + } + + #[test] + fn serve_http_refuses_a_keyed_container_that_answers_a_route_without_the_key() { + if in_subprocess() { + let port = spawn_fake_server(SERVED, false); + let state_path = data_dir().join("svc.json"); + let error = serve_http(serve_request("svc-open-route", port, state_path.clone())) + .expect_err("a keyed container that answers keyless calls must not serve"); + let text = format!("{error:#}"); + assert!(text.contains("POST /tokenize"), "{text}"); + assert!(text.contains("HTTP 200"), "{text}"); + assert_eq!(status_of(&state_path).as_deref(), Some("failed")); + return; + } + let key_file = endpoint_key_file("open"); + let fixture = run_with_fake_docker( + "process::subprocess_tests::serve_http_refuses_a_keyed_container_that_answers_a_route_without_the_key", + &[ + LABELLED_IMAGE, + ("FAKE_DOCKER_IMAGE_PRESENT_EXIT", "0"), + // Ends exited, so a start that skipped the check would return + // instead of supervising the fake container forever. + ("FAKE_DOCKER_INSPECT_SEQUENCE", "missing,true 0,false 0"), + ( + rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, + key_file.to_str().unwrap(), + ), + ], + ); + std::fs::remove_file(&key_file).ok(); + assert!( + fixture + .argv() + .contains(&"rm -f rocm-aim-svc-open-route".to_owned()), + "the unprotected container must be removed: {:?}", + fixture.argv() + ); + } + + #[test] + fn serve_http_serves_a_keyed_container_that_refuses_keyless_calls() { + if in_subprocess() { + let port = spawn_fake_server(SERVED, true); + let state_path = data_dir().join("svc.json"); + serve_http(serve_request("svc-guarded", port, state_path.clone())).unwrap(); + assert_eq!(status_of(&state_path).as_deref(), Some("stopped")); + let state = crate::state::read_service_state(&state_path).unwrap(); + assert_eq!(state["served_model_id"], SERVED); + return; + } + let key_file = endpoint_key_file("guarded"); + run_with_fake_docker( + "process::subprocess_tests::serve_http_serves_a_keyed_container_that_refuses_keyless_calls", + &[ + LABELLED_IMAGE, + ("FAKE_DOCKER_IMAGE_PRESENT_EXIT", "0"), + ( + "FAKE_DOCKER_INSPECT_SEQUENCE", + "missing,true 0,true 0,false 0", + ), ( rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file.to_str().unwrap(), @@ -1299,8 +1539,6 @@ mod subprocess_tests { ], ); std::fs::remove_file(&key_file).ok(); - assert!(!fixture.argv().join("\n").contains("s3cret")); - assert_eq!(fixture.env_copy().as_deref(), Some("VLLM_API_KEY=s3cret\n")); } #[test] diff --git a/engines/aim/src/registry.rs b/engines/aim/src/registry.rs index 29e0be5d0..cb95cc465 100644 --- a/engines/aim/src/registry.rs +++ b/engines/aim/src/registry.rs @@ -22,6 +22,9 @@ pub(crate) struct AimImageMetadata { pub accelerator_type: Option, pub pytorch_rocm_arch: Vec, pub canonical_name: Option, + /// Every `KEY=VALUE` entry of the image's `Env`, for values a launch has + /// to extend rather than replace (`PYTHONPATH`, `AIM_ENGINE_ARGS`). + pub env: std::collections::BTreeMap, } /// Normalizes a user- or catalog-supplied image reference so the registry @@ -139,8 +142,20 @@ pub(crate) fn fetch_image_metadata(image_ref: &str) -> Result /// Metadata from the local daemon's copy of the image, or `None` if the image /// isn't present locally. pub(crate) fn local_image_metadata(image_ref: &str) -> Option { - let config = crate::docker::image_config(image_ref).ok().flatten()?; - Some(parse_image_config(&config)) + read_local_image_metadata(image_ref).ok() +} + +/// [`local_image_metadata`], saying why it could not be read: `docker image +/// inspect` failed to run, or answered without a config it could parse. +pub(crate) fn read_local_image_metadata(image_ref: &str) -> Result { + // `null` is what inspect prints for an image with no config at all, which + // tells a keyed launch no more about the image's env than no answer does. + let config = crate::docker::image_config(image_ref)? + .filter(|config| !config.is_null()) + .with_context(|| { + format!("`docker image inspect {image_ref}` returned no readable `.Config`") + })?; + Ok(parse_image_config(&config)) } /// Parses an image `Config` object (`{"Env": [...], "Labels": {...}}`), the @@ -156,6 +171,7 @@ fn parse_image_config(config: &Value) -> AimImageMetadata { .unwrap_or_default(); for entry in env_entries { if let Some((key, value)) = entry.split_once('=') { + metadata.env.insert(key.to_owned(), value.to_owned()); match key { "AIM_ACCELERATOR_TYPE" => metadata.accelerator_type = Some(value.to_owned()), "PYTORCH_ROCM_ARCH" => { @@ -291,5 +307,9 @@ mod tests { metadata.canonical_name.as_deref(), Some("Llama-3.2-1B-Instruct") ); + assert_eq!( + metadata.env.get("PATH").map(String::as_str), + Some("/usr/bin") + ); } } diff --git a/engines/vllm/src/endpoint_auth.rs b/engines/vllm/src/endpoint_auth.rs new file mode 100644 index 000000000..9776ee931 --- /dev/null +++ b/engines/vllm/src/endpoint_auth.rs @@ -0,0 +1,195 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Endpoint API-key enforcement for the native vLLM server. +//! +//! Setting `VLLM_API_KEY` alone does not protect a vLLM endpoint: vLLM's own +//! check guards only a fixed set of path prefixes (`/v1` and a few others), +//! while the server also registers inference-capable routes outside them — +//! `/invocations`, `/tokenize`, `/detokenize` and more. So whenever an endpoint +//! key is in force, the server is additionally started with +//! `--middleware rocm_cli_vllm_auth.RequireEndpointApiKey`: the ASGI middleware +//! shared with the AIM engine (`rocm_engine_protocol::endpoint_auth`), written +//! next to the engine's data at launch, that requires the key on every route +//! except `GET`/`HEAD /health` and CORS preflights. +//! +//! vLLM imports the middleware by module path, so the directory it is written +//! to is prepended to the server's `PYTHONPATH`. A failed import aborts vLLM's +//! startup, so a launch that asked for a key fails closed rather than coming up +//! with only the partial check. + +use anyhow::Result; +use rocm_engine_protocol::endpoint_auth::{MIDDLEWARE_KEY_ENV, write_middleware_module}; +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::process::Command as ProcessCommand; + +/// Directory the middleware module is written to under the engine's data +/// directory. Owned by the invoking user like the rest of that tree, so only +/// they can replace the code vLLM imports. +pub(crate) fn module_dir(paths: &rocm_core::AppPaths) -> PathBuf { + paths.engine_dir(crate::ENGINE_NAME).join("python") +} + +/// Require `api_key` on every route of the vLLM server `command` will start. +/// +/// Sets the key, writes the middleware module into `module_dir` and prepends +/// that directory to `PYTHONPATH`. Call after any other `PYTHONPATH` change +/// (see `apply_therock_env`): an entry `command` already sets is extended +/// rather than replaced. The `--middleware` argument itself comes from +/// `rocm_engine_protocol::endpoint_auth::middleware_args` via +/// `vllm_serve_args`, so the argv stays a pure function of the launch request. +pub(crate) fn protect_endpoint( + command: &mut ProcessCommand, + api_key: &str, + module_dir: &Path, +) -> Result<()> { + write_middleware_module(module_dir)?; + command.env(MIDDLEWARE_KEY_ENV, api_key); + let pythonpath = middleware_pythonpath( + module_dir, + command_env(command, "PYTHONPATH"), + std::env::var_os("PYTHONPATH"), + )?; + command.env("PYTHONPATH", pythonpath); + Ok(()) +} + +/// How a [`ProcessCommand`] treats one environment variable. +#[derive(Debug, PartialEq, Eq)] +enum EnvOverride { + /// The command leaves it alone; the child inherits this process's value. + Inherited, + /// The command removes it (`env_remove`). + Removed, + /// The command sets it to this value. + Set(OsString), +} + +/// The server's `PYTHONPATH`: `module_dir` first, then whatever the server +/// would otherwise have seen — the value the command sets, nothing if it +/// removes the variable, or `inherited` when it leaves it alone. Dropping the +/// inherited value would silently change which packages the server imports. +fn middleware_pythonpath( + module_dir: &Path, + command_value: EnvOverride, + inherited: Option, +) -> Result { + let current = match command_value { + EnvOverride::Inherited => inherited, + EnvOverride::Removed => None, + EnvOverride::Set(value) => Some(value), + }; + crate::process::prepend_path_entries(&[module_dir.to_path_buf()], current) +} + +/// How `command` treats the environment variable `key`. +fn command_env(command: &ProcessCommand, key: &str) -> EnvOverride { + command + .get_envs() + .find_map(|(name, value)| (name == key).then_some(value)) + .map_or(EnvOverride::Inherited, |value| match value { + Some(value) => EnvOverride::Set(value.to_owned()), + None => EnvOverride::Removed, + }) +} + +/// The value `command` sets `key` to, if it sets one. Shared by this module's +/// tests and the launch-command tests in `process.rs`. +#[cfg(test)] +pub(crate) fn env_of(command: &ProcessCommand, key: &str) -> Option { + match command_env(command, key) { + EnvOverride::Set(value) => Some(value), + EnvOverride::Inherited | EnvOverride::Removed => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rocm_engine_protocol::endpoint_auth::{MIDDLEWARE_MODULE_NAME, MIDDLEWARE_MODULE_SOURCE}; + use std::fs; + + fn entries(value: &OsString) -> Vec { + std::env::split_paths(value).collect() + } + + #[test] + fn middleware_pythonpath_keeps_the_inherited_value_when_the_command_sets_none() -> Result<()> { + let module_dir = PathBuf::from("/data/python"); + let composed = middleware_pythonpath( + &module_dir, + EnvOverride::Inherited, + Some("/user/site".into()), + )?; + assert_eq!( + entries(&composed), + vec![module_dir, PathBuf::from("/user/site")], + "the server must still see the PYTHONPATH it would have inherited" + ); + Ok(()) + } + + #[test] + fn middleware_pythonpath_prefers_the_command_value_over_the_inherited_one() -> Result<()> { + let module_dir = PathBuf::from("/data/python"); + let composed = middleware_pythonpath( + &module_dir, + EnvOverride::Set("/sdk/amd_smi".into()), + Some("/user/site".into()), + )?; + assert_eq!( + entries(&composed), + vec![module_dir, PathBuf::from("/sdk/amd_smi")] + ); + Ok(()) + } + + #[test] + fn middleware_pythonpath_honours_a_command_that_removes_the_variable() -> Result<()> { + let module_dir = PathBuf::from("/data/python"); + let composed = + middleware_pythonpath(&module_dir, EnvOverride::Removed, Some("/user/site".into()))?; + assert_eq!(entries(&composed), vec![module_dir]); + Ok(()) + } + + #[test] + fn command_env_distinguishes_inherited_removed_and_set() { + let mut command = ProcessCommand::new("vllm"); + assert_eq!(command_env(&command, "PYTHONPATH"), EnvOverride::Inherited); + command.env_remove("PYTHONPATH"); + assert_eq!(command_env(&command, "PYTHONPATH"), EnvOverride::Removed); + command.env("PYTHONPATH", "/x"); + assert_eq!( + command_env(&command, "PYTHONPATH"), + EnvOverride::Set("/x".into()) + ); + } + + #[test] + fn protect_endpoint_sets_key_and_makes_the_middleware_importable() -> Result<()> { + let temp = tempfile::tempdir()?; + let module_dir = temp.path().join("python"); + let mut command = ProcessCommand::new("vllm"); + command.env("PYTHONPATH", "/opt/existing"); + + protect_endpoint(&mut command, "secret-key", &module_dir)?; + + assert_eq!( + env_of(&command, MIDDLEWARE_KEY_ENV), + Some("secret-key".into()) + ); + let pythonpath = env_of(&command, "PYTHONPATH").expect("PYTHONPATH set"); + let entries: Vec = std::env::split_paths(&pythonpath).collect(); + assert_eq!( + entries, + vec![module_dir.clone(), PathBuf::from("/opt/existing")], + "the middleware directory goes first and an earlier entry is kept" + ); + let written = fs::read_to_string(module_dir.join(format!("{MIDDLEWARE_MODULE_NAME}.py")))?; + assert_eq!(written, MIDDLEWARE_MODULE_SOURCE); + Ok(()) + } +} diff --git a/engines/vllm/src/lib.rs b/engines/vllm/src/lib.rs index 80ad31eca..836976797 100644 --- a/engines/vllm/src/lib.rs +++ b/engines/vllm/src/lib.rs @@ -18,6 +18,7 @@ use std::io::{Read, Write}; use std::path::PathBuf; mod discover_builds; +mod endpoint_auth; mod install; mod process; mod repair; diff --git a/engines/vllm/src/process.rs b/engines/vllm/src/process.rs index 87f3d4f9a..821559804 100644 --- a/engines/vllm/src/process.rs +++ b/engines/vllm/src/process.rs @@ -235,25 +235,19 @@ the ROCm SDK's bundled numa uses renamed symbol versions and cannot satisfy it, .with_context(|| format!("failed to create {}", parent.display()))?; } - let mut command = ProcessCommand::new(&runtime.command); - command - .args(vllm_serve_args( - &request.model_ref, - &request.host, - request.port, - vllm_enforce_eager_enabled(), - request.engine_recipe.as_ref(), - )) - .stdin(Stdio::null()); - // When `rocm serve` protects a public endpoint, the key arrives via the - // environment / key file (never argv). Hand it to vLLM as `VLLM_API_KEY` so - // its OpenAI server rejects unauthenticated requests; passing it through the - // env keeps it out of the process table. - if let Some(api_key) = rocm_engine_protocol::resolve_endpoint_api_key() { - command.env("VLLM_API_KEY", api_key); - } - apply_therock_env(&mut command, runtime)?; - rocm_engine_protocol::apply_gpu_visibility(&mut command, &request.gpu_indices); + // When `rocm serve` protects an endpoint, the key arrives via the key file + // (never argv). It reaches vLLM through the environment, which keeps it out + // of the process table, and is enforced on every route by the middleware + // `endpoint_auth` installs: vLLM's own `VLLM_API_KEY` check alone leaves + // routes such as `/invocations` open. + let api_key = rocm_engine_protocol::resolve_endpoint_api_key(); + let mut command = vllm_server_command( + request, + runtime, + api_key.as_deref(), + || Ok(crate::endpoint_auth::module_dir(&AppPaths::discover()?)), + vllm_enforce_eager_enabled(), + )?; if let Some(log_path) = log_path { let log = fs::File::create(log_path) .with_context(|| format!("failed to create {}", log_path.display()))?; @@ -274,6 +268,40 @@ the ROCm SDK's bundled numa uses renamed symbol versions and cannot satisfy it, } }) } +/// The `vllm serve` command for `request`, short of where its output goes: +/// argv, TheRock environment and GPU visibility, plus, with an `api_key`, the +/// endpoint-key protection: the `--middleware` argument, `VLLM_API_KEY`, and +/// the middleware module written to `module_dir()` and put at the front of +/// `PYTHONPATH`. `--middleware` depends on the other two: vLLM has to import +/// the module from `PYTHONPATH`, and the middleware refuses to construct +/// without `VLLM_API_KEY`. So all three are built in one place and asserted +/// together by one test. `module_dir` is consulted only for a keyed launch. +fn vllm_server_command( + request: &ServeHttpRequest, + runtime: &VllmRuntime, + api_key: Option<&str>, + module_dir: impl FnOnce() -> Result, + enforce_eager: bool, +) -> Result { + let mut command = ProcessCommand::new(&runtime.command); + command + .args(vllm_serve_args( + &request.model_ref, + &request.host, + request.port, + enforce_eager, + request.engine_recipe.as_ref(), + api_key.is_some(), + )) + .stdin(Stdio::null()); + apply_therock_env(&mut command, runtime)?; + // After `apply_therock_env`, which may set its own `PYTHONPATH` entry. + if let Some(api_key) = api_key { + crate::endpoint_auth::protect_endpoint(&mut command, api_key, &module_dir()?)?; + } + rocm_engine_protocol::apply_gpu_visibility(&mut command, &request.gpu_indices); + Ok(command) +} /// Explain a spawn that failed on a script whose `#!` interpreter is gone. /// /// The kernel reports the missing *interpreter* as ENOENT against the *script*, @@ -516,7 +544,8 @@ fn apply_vllm_rocm10_discover_env( command.env("FLASH_ATTENTION_TRITON_AMD_ENABLE", "TRUE"); Ok(()) } -/// Full argument vector for `vllm serve`, as spawned by [`spawn_vllm_server`]. +/// Full argument vector for `vllm serve`, as built into the launch command by +/// [`vllm_server_command`]. /// /// Kept as a pure function so the exact argv can be asserted in tests: memory /// and graph-mode flags materially change how much VRAM vLLM claims, and a @@ -526,12 +555,16 @@ fn apply_vllm_rocm10_discover_env( /// supply a default — vLLM's own default applies unless the user asks for a /// specific fraction via `rocm serve --gpu-memory-utilization`, which arrives /// here through the engine recipe's `required_flags`. +/// +/// `require_api_key` adds the `--middleware` that enforces the endpoint key on +/// every route (see [`crate::endpoint_auth`]). fn vllm_serve_args( model_ref: &str, host: &str, port: u16, enforce_eager: bool, engine_recipe: Option<&EngineRecipeHint>, + require_api_key: bool, ) -> Vec { let mut args = vec![ "serve".to_owned(), @@ -549,6 +582,9 @@ fn vllm_serve_args( args.push("--enforce-eager".to_owned()); } args.extend(engine_recipe_launch_args(engine_recipe)); + if require_api_key { + args.extend(rocm_engine_protocol::endpoint_auth::middleware_args()); + } args } /// Whether to launch vLLM with `--enforce-eager` (CUDA graphs disabled). @@ -583,7 +619,10 @@ fn resolve_vllm_ready_timeout(override_value: Option) -> Duration { .map_or(DEFAULT_VLLM_READY_TIMEOUT, Duration::from_secs) } -fn prepend_path_entries(entries: &[PathBuf], current: Option) -> Result { +pub(crate) fn prepend_path_entries( + entries: &[PathBuf], + current: Option, +) -> Result { let mut parts = Vec::new(); for entry in entries { if !entry.as_os_str().is_empty() && !parts.iter().any(|part: &PathBuf| part == entry) { @@ -815,7 +854,7 @@ mod tests { #[test] fn vllm_serve_args_omit_gpu_memory_utilization_without_override() { - let args = vllm_serve_args("facebook/opt-125m", "127.0.0.1", 8000, false, None); + let args = vllm_serve_args("facebook/opt-125m", "127.0.0.1", 8000, false, None, false); assert_eq!( args, @@ -842,7 +881,14 @@ mod tests { ..EngineRecipeHint::default() }; - let args = vllm_serve_args("facebook/opt-125m", "127.0.0.1", 8000, true, Some(&hint)); + let args = vllm_serve_args( + "facebook/opt-125m", + "127.0.0.1", + 8000, + true, + Some(&hint), + false, + ); let position = args .iter() @@ -852,6 +898,178 @@ mod tests { assert!(args.contains(&"--enforce-eager".to_owned())); } #[test] + fn vllm_serve_args_install_the_auth_middleware_when_a_key_is_required() { + let args = vllm_serve_args("facebook/opt-125m", "0.0.0.0", 8000, false, None, true); + + let position = args + .iter() + .position(|arg| arg == "--middleware") + .expect("a keyed endpoint must be launched with the auth middleware"); + assert_eq!( + args.get(position + 1).map(String::as_str), + Some(rocm_engine_protocol::endpoint_auth::MIDDLEWARE_IMPORT_PATH) + ); + assert!( + !args.iter().any(|arg| arg.contains("api-key")), + "the key itself must never reach argv: {args:?}" + ); + } + #[test] + fn vllm_serve_args_omit_the_auth_middleware_without_a_key() { + let args = vllm_serve_args("facebook/opt-125m", "127.0.0.1", 8000, false, None, false); + assert!( + !args.iter().any(|arg| arg == "--middleware"), + "an unkeyed endpoint must not load the middleware, which refuses to construct \ + without a key: {args:?}" + ); + } + fn keyed_launch_request() -> ServeHttpRequest { + ServeHttpRequest { + service_id: "svc-vllm".to_owned(), + model_ref: "facebook/opt-125m".to_owned(), + host: "0.0.0.0".to_owned(), + port: 8000, + device_policy: DevicePolicy::GpuRequired, + gpu_indices: vec![1], + runtime_id: None, + env_id: None, + state_path: PathBuf::from("state.json"), + log_path: None, + engine_recipe: None, + } + } + #[test] + fn keyed_vllm_server_command_installs_middleware_key_and_pythonpath_together() -> Result<()> { + let temp = tempfile::tempdir()?; + let module_dir = temp.path().join("python"); + // A ROCm 10 discover runtime, whose TheRock environment sets its own + // `PYTHONPATH` entry (`amd_smi` in the venv). The middleware directory + // has to be prepended after that, or the TheRock value replaces it. + let venv = temp.path().join("venv"); + let site_packages = venv.join("lib").join("python3.12").join("site-packages"); + fs::create_dir_all(&site_packages)?; + let runtime = test_vllm_runtime( + Some(venv.join("bin").join("python")), + Some("10.0.0".to_owned()), + ); + let command = vllm_server_command( + &keyed_launch_request(), + &runtime, + Some("secret-key"), + || Ok(module_dir.clone()), + false, + )?; + + let args: Vec = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect(); + let position = args + .iter() + .position(|arg| arg == "--middleware") + .expect("a keyed launch must pass the auth middleware to vllm serve"); + assert_eq!( + args.get(position + 1).map(String::as_str), + Some(rocm_engine_protocol::endpoint_auth::MIDDLEWARE_IMPORT_PATH) + ); + assert!( + !args.iter().any(|arg| arg.contains("secret-key")), + "the key must never reach argv: {args:?}" + ); + assert!( + !args.iter().any(|arg| arg == "--enforce-eager"), + "enforce_eager was false: {args:?}" + ); + assert_eq!( + crate::endpoint_auth::env_of(&command, "VLLM_API_KEY"), + Some("secret-key".into()), + "the middleware and vLLM's own check both read the key from VLLM_API_KEY" + ); + let pythonpath = crate::endpoint_auth::env_of(&command, "PYTHONPATH") + .expect("a keyed launch must put the middleware directory on PYTHONPATH"); + let entries: Vec = std::env::split_paths(&pythonpath).collect(); + let amd_smi = site_packages + .join("_rocm_sdk_core") + .join("share") + .join("amd_smi"); + assert_eq!( + entries.get(..2), + Some(&[module_dir.clone(), amd_smi][..]), + "vLLM imports the middleware by module name, so its directory goes first, \ + ahead of the runtime's own entry, which is kept: {entries:?}" + ); + assert_eq!( + crate::endpoint_auth::env_of(&command, "HIP_VISIBLE_DEVICES"), + Some("1".into()), + "the GPU the request selected must still be applied to a keyed launch" + ); + assert!( + module_dir.join("rocm_cli_vllm_auth.py").is_file(), + "the middleware module must be written where PYTHONPATH points" + ); + Ok(()) + } + #[test] + fn vllm_server_command_forwards_enforce_eager() -> Result<()> { + let command = vllm_server_command( + &keyed_launch_request(), + &VllmRuntime::for_test(), + None, + || panic!("an unkeyed launch must not need the middleware directory"), + true, + )?; + assert!( + command.get_args().any(|arg| arg == "--enforce-eager"), + "enforce_eager was true: {:?}", + command.get_args().collect::>() + ); + Ok(()) + } + #[test] + fn unkeyed_vllm_server_command_carries_no_key_protection() -> Result<()> { + // The same TheRock runtime as the keyed test, so `PYTHONPATH` has a value + // of its own that a stray middleware entry would be prepended to. + let temp = tempfile::tempdir()?; + let venv = temp.path().join("venv"); + let site_packages = venv.join("lib").join("python3.12").join("site-packages"); + fs::create_dir_all(&site_packages)?; + let runtime = test_vllm_runtime( + Some(venv.join("bin").join("python")), + Some("10.0.0".to_owned()), + ); + let command = vllm_server_command( + &keyed_launch_request(), + &runtime, + None, + || panic!("an unkeyed launch must not need the middleware directory"), + false, + )?; + assert!( + !command.get_args().any(|arg| arg == "--middleware"), + "an unkeyed launch must not load the middleware, which refuses to start without a key" + ); + assert_eq!(crate::endpoint_auth::env_of(&command, "VLLM_API_KEY"), None); + let pythonpath = crate::endpoint_auth::env_of(&command, "PYTHONPATH") + .expect("the TheRock runtime sets its own PYTHONPATH entry"); + let entries: Vec = std::env::split_paths(&pythonpath).collect(); + let amd_smi = site_packages + .join("_rocm_sdk_core") + .join("share") + .join("amd_smi"); + // The runtime's own value: its entry ahead of whatever PYTHONPATH this + // process inherited. + let expected: Vec = std::env::split_paths(&prepend_path_entries( + &[amd_smi], + std::env::var_os("PYTHONPATH"), + )?) + .collect(); + assert_eq!( + entries, expected, + "an unkeyed launch keeps the runtime's PYTHONPATH exactly, with no middleware directory" + ); + Ok(()) + } + #[test] fn resolve_model_echoes_matching_engine_recipe() -> Result<()> { let hint = test_engine_recipe(ENGINE_NAME, ENGINE_RECIPE_CONTRACT_VERSION); let response = resolve_model_response(ResolveModelRequest { diff --git a/tests/e2e-cucumber/features/diagnose.feature b/tests/e2e-cucumber/features/diagnose.feature index 776e69a01..e7403f40b 100644 --- a/tests/e2e-cucumber/features/diagnose.feature +++ b/tests/e2e-cucumber/features/diagnose.feature @@ -909,7 +909,8 @@ Feature: Diagnosing failures and listing fixes # paste is diagnosed (not investigated further; the console is started with # cursor inheritance, which waits on a reply the screen emulator does not # send). That leaves the Windows half of the hint (Ctrl-Z then Enter) without - # an e2e check; docs/manual-testing.md section 13 covers it by hand. + # an e2e check; docs/manual-testing.md's "Pasting An Error Into `rocm diagnose`" + # section covers it by hand. @id:diagnose-a-paste-at-the-terminal-is-explained-and-diagnosed @requires-os:linux Scenario: diagnose-60 - Pasting the error at the terminal is explained, and the paste is diagnosed when the user ends it Given a pasted error containing quotes, a command substitution and a leading minus sign diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 096bd0825..138094f54 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -365,14 +365,15 @@ Feature: Model serving # verified on (see `HostCapability::engine_available`'s "aim" arm — Radeon # gfx110X/120X and Strix Halo gfx1151, not Instinct); a Strix Halo # self-hosted runner with Docker is where this actually runs. - # `@nightly` keeps both AIM scenarios off the pull-request lanes: each one - # pulls the image and a 9B checkpoint on a cold runner and may wait out the - # full 1200s readiness cap, so the pair on the Strix Ubuntu job (~25 min - # without them, 45 min cap) ended every run at the cap. That job now runs + # `@nightly` keeps the AIM scenarios (these two and serve-62) off the + # pull-request lanes: each one pulls the image and a 9B checkpoint on a cold + # runner and may wait out the full 1200s readiness cap, so these two on the + # Strix Ubuntu job (~25 min without them, 45 min cap) ended every run at + # the cap. That job now runs # only on push, dispatch and nightly, never on pull_request. They run only when # E2E_INCLUDE_NIGHTLY is set (nightly.yml, or an include_nightly dispatch of # e2e-selfhosted.yml) on a host that has Docker and a supported GPU. - # On a host without Docker both skip (the nightly job logs whether Docker is + # On a host without Docker they skip (the nightly job logs whether Docker is # reachable). serve-27's stop failure on Strix Halo Ubuntu (rocm-cli#1058) # passed on the 2026-11-15 and 2026-11-16 nightlies, so its xfail row is gone. # `@serve-timeout:1200` @@ -980,3 +981,47 @@ Feature: Model serving And the deployment summary names the logs command for that service And the launch's service record is left as ready And no background automation daemon was left behind + + # An API key on a vLLM endpoint (`--require-api-key`, a public bind, or + # `rocm remote serve`) has to cover the whole server, not only the OpenAI `/v1` + # API: vLLM's own key check guards a fixed set of path prefixes, while the + # server also answers inference and tokenizer routes outside them. rocm-cli + # starts vLLM with a middleware that demands the key on every route except + # `GET`/`HEAD /health` and CORS preflights. The middleware itself is + # unit-tested on every PR (`endpoint_auth` in the engine-protocol crate, + # where both engines take it from); this proves it is actually wired into a + # real vLLM server. A full extra vLLM + # serve, so `@nightly` rather than per-PR: it runs after merge in the nightly + # runs on Instinct GPUs, which serve with vLLM: the MI300X job in + # `nightly.yml` and the shared GPU queue's Instinct lanes (MI350P among them). + # Not `@merge-queue`: the merge queue runs no lane where vLLM is the engine + # (the MI300X lane does not run on `merge_group`), so under that tag it + # skipped everywhere. + @id:serve-vllm-key-guards-every-route @requires-gpu @requires-engine:vllm @nightly + Scenario: serve-61 - A key-protected vLLM endpoint refuses keyless requests outside the OpenAI API + Given a managed runtime is active + And a model is being served on GPU behind an API key + When a client calls the server's routes without the key + Then every route outside the OpenAI API refuses the call as unauthorized + And the health check answers without the key + When the client repeats those calls with the key + Then every one of those calls succeeds + + # The AIM engine's container bundles vLLM, so the same gap applies: its + # `VLLM_API_KEY` check guards only vLLM's own path prefixes. rocm-cli mounts + # the same middleware into the container and adds it through the AIM + # runtime's `AIM_ENGINE_ARGS`, then refuses to serve unless a keyless call to + # `/tokenize` is refused. The env, mount and refusal are unit-tested on every + # PR (`endpoint_auth` and the keyed `run_container`/`serve_http` tests in the + # AIM engine crate); this proves a real AIM image actually loads the + # middleware. `@nightly` for the same reason as serve-26: it pulls the image + # and a 9B checkpoint on a cold runner. + @id:serve-aim-key-guards-every-route @requires-gpu @requires-engine:aim @serve-timeout:1200 @nightly + Scenario: serve-62 - A key-protected AIM endpoint refuses keyless requests outside the OpenAI API + Given a managed runtime is active + And a model is being served on aim behind an API key + When a client calls the server's routes without the key + Then every route outside the OpenAI API refuses the call as unauthorized + And the health check answers without the key + When the client repeats those calls with the key + Then every one of those calls succeeds diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 6771d09dc..aa97badcd 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1198,14 +1198,14 @@ serve_timeout_secs = 90 #[test] fn aim_serve_scenarios_stay_off_the_per_pr_lanes() { - // serve-26/27 pull the AIM image and a 9B checkpoint on a cold runner and + // serve-26/27/62 pull the AIM image and a 9B checkpoint on a cold runner and // may each wait out a 1200s readiness cap. Run on every PR on the Strix // Ubuntu pool (before that lane left the PR path) they pushed the job // (~25 min otherwise) into its 45 min cap on every run. They must // resolve to Skip unless the nightly opt-in is set, and must still be // reachable (not Skip for the nightly reason) when it is. let m = Expectations::default(); - for prefix in ["serve-26 -", "serve-27 -"] { + for prefix in ["serve-26 -", "serve-27 -", "serve-62 -"] { let d = ScenarioDecl::from_tags(&model_serving_tags(prefix)); assert_eq!(d.requires_engine.as_deref(), Some("aim"), "{prefix}"); assert!(d.nightly, "{prefix} must be @nightly"); diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index ae41672ba..d1a807ab1 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -173,6 +173,9 @@ pub struct E2eWorld { /// from. Consumed by [`run_rocm_with_scenario_env`] like `command_env`; `None` /// leaves the harness's own working directory. pub command_dir: Option, + /// `(request, HTTP status)` for each server route a step called directly, + /// in call order, for a later Then step to judge. + pub route_statuses: Vec<(String, u16)>, /// The interactive dash/chat TUI spawned under a pseudo-terminal for this /// scenario, if any (see `e2e::tui_driver`). Torn down in `Drop` before the /// mock server and isolated directory so the child process never outlives @@ -402,6 +405,7 @@ impl Default for E2eWorld { expect_xfail: false, command_env: Vec::new(), command_dir: None, + route_statuses: Vec::new(), tui: None, chat_use_mock: false, comfyui_baseline_torch_stack: None, @@ -981,6 +985,18 @@ pub fn run_rocm_bounded( world: &E2eWorld, args: &[&str], timeout: std::time::Duration, +) -> (String, String, i32) { + run_rocm_bounded_with_env(world, args, &[], timeout) +} + +/// Like [`run_rocm_bounded`], but sets `envs` on the child on top of the +/// scenario's isolated env, for a value that must not appear in the recorded +/// argv (an endpoint API key). +pub fn run_rocm_bounded_with_env( + world: &E2eWorld, + args: &[&str], + envs: &[(&str, &str)], + timeout: std::time::Duration, ) -> (String, String, i32) { use std::process::Stdio; @@ -991,6 +1007,7 @@ pub fn run_rocm_bounded( let mut cmd = std::process::Command::new(&binary); cmd.args(args); world.isolate_cmd(&mut cmd); + cmd.envs(envs.iter().copied()); cmd.stdin(Stdio::null()); cmd.stdout(Stdio::from( std::fs::File::create(&out_path).expect("stdout file"), diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 990bb6b18..38c7c81a1 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -95,9 +95,25 @@ pub(crate) async fn model_is_ready( expect_model: Option<&str>, timeout_secs: u64, ) -> bool { + model_is_ready_with_key(models_url, expect_model, timeout_secs, None).await +} + +/// [`model_is_ready`] against an endpoint protected by `api_key`, which a +/// keyless listing would only ever see refused. +async fn model_is_ready_with_key( + models_url: &str, + expect_model: Option<&str>, + timeout_secs: u64, + api_key: Option<&str>, +) -> bool { + let client = reqwest::Client::new(); let deadline = Instant::now() + Duration::from_secs(timeout_secs); while Instant::now() < deadline { - if let Ok(resp) = reqwest::get(models_url).await + let mut request = client.get(models_url); + if let Some(key) = api_key { + request = request.bearer_auth(key); + } + if let Ok(resp) = request.send().await && resp.status().is_success() { match expect_model { @@ -214,20 +230,47 @@ async fn serve_and_wait( args: &[&str], model: &str, ready_substr: &str, +) -> String { + serve_and_wait_with_key(world, args, model, ready_substr, None).await +} + +/// [`serve_and_wait`], protecting the endpoint with `api_key` when one is +/// given: `--require-api-key` is appended and the key travels in +/// `ROCM_SERVE_API_KEY`, never argv, which is recorded verbatim in the +/// uploaded `results/commands.jsonl`. +async fn serve_and_wait_with_key( + world: &mut E2eWorld, + args: &[&str], + model: &str, + ready_substr: &str, + api_key: Option<&str>, ) -> String { let timeout_secs = serve_timeout_for(world); let device_state = ensure_serve_port_free().await; + // The key goes through `ROCM_SERVE_API_KEY`, as `rocm serve --help` + // advises, not `--api-key`: argv is recorded verbatim in the uploaded + // `results/commands.jsonl`. + let mut args = args.to_vec(); + let mut envs = Vec::new(); + if let Some(key) = api_key { + args.push("--require-api-key"); + envs.push(("ROCM_SERVE_API_KEY", key)); + } + let args = args.as_slice(); // Bounded: a `rocm serve` that never returns must fail this scenario, not // stall the lane until the job is cancelled. The bound is the readiness // timeout plus a margin, so a slow cold start fails through the CLI's own // diagnostic rather than a harness kill; readiness polling keeps the plain // timeout. - let (stdout, stderr, rc) = crate::run_rocm_bounded( + let (stdout, stderr, rc) = crate::run_rocm_bounded_with_env( world, args, + &envs, e2e_cucumber::serve_log::serve_cli_budget(timeout_secs), ); - if rc == 0 && model_is_ready(&MODELS_URL, Some(ready_substr), timeout_secs).await { + if rc == 0 + && model_is_ready_with_key(&MODELS_URL, Some(ready_substr), timeout_secs, api_key).await + { world.endpoint = Some(SERVE_BASE_URL.to_string()); world.model_name = Some(model.to_string()); return stdout; @@ -628,6 +671,145 @@ async fn setup_gpu_model(world: &mut E2eWorld) { serve_gpu_model(world).await; } +/// The key the keyed-serve scenario protects its endpoint with. A fixed value: +/// what that scenario checks is whether the key is enforced, not how it is minted. +const PROTECTED_ENDPOINT_KEY: &str = "e2e-protected-endpoint-key-7f3a91c2"; + +#[given("a model is being served on GPU behind an API key")] +async fn setup_gpu_model_behind_key(world: &mut E2eWorld) { + serve_gpu_model_with_key(world, Some(PROTECTED_ENDPOINT_KEY)).await; +} + +/// Server routes outside the OpenAI `/v1` API that can run the model, read its +/// tokenizer or report on it. Each must demand the key like `/v1` does; the +/// bodies are valid requests, so a refusal can only come from authentication. +fn routes_outside_the_openai_api( + model: &str, +) -> Vec<(reqwest::Method, &'static str, serde_json::Value)> { + vec![ + ( + reqwest::Method::POST, + "/invocations", + serde_json::json!({ + "model": model, + "messages": [{"role": "user", "content": "Hello"}], + "max_tokens": 8, + }), + ), + ( + reqwest::Method::POST, + "/tokenize", + serde_json::json!({"model": model, "prompt": "Hello"}), + ), + ( + reqwest::Method::POST, + "/detokenize", + serde_json::json!({"model": model, "tokens": [9707]}), + ), + (reqwest::Method::GET, "/metrics", serde_json::Value::Null), + ] +} + +/// The liveness probe, the one route the middleware answers without the key. +const HEALTH_ROUTE: &str = "GET /health"; + +/// Call every route in [`routes_outside_the_openai_api`] and then `GET /health`, +/// with or without the key, recording each status in `world.route_statuses`. +async fn call_routes_outside_the_openai_api(world: &mut E2eWorld, api_key: Option<&str>) { + let model = world.model_name.clone().expect("no model served"); + let client = reqwest::Client::builder() + .timeout(Duration::from_mins(2)) + .build() + .expect("http client"); + let base = format!("http://127.0.0.1:{SERVE_PORT}"); + world.route_statuses.clear(); + for (method, path, body) in routes_outside_the_openai_api(&model) { + let mut request = client.request(method.clone(), format!("{base}{path}")); + if !body.is_null() { + request = request.json(&body); + } + if let Some(key) = api_key { + request = request.bearer_auth(key); + } + let status = request + .send() + .await + .unwrap_or_else(|e| panic!("{method} {base}{path} failed: {e}")) + .status() + .as_u16(); + world + .route_statuses + .push((format!("{method} {path}"), status)); + } + let mut health = client.get(format!("{base}/health")); + if let Some(key) = api_key { + health = health.bearer_auth(key); + } + let status = health + .send() + .await + .unwrap_or_else(|e| panic!("GET {base}/health failed: {e}")) + .status() + .as_u16(); + world.route_statuses.push((HEALTH_ROUTE.to_owned(), status)); +} + +#[when("a client calls the server's routes without the key")] +async fn client_calls_routes_without_key(world: &mut E2eWorld) { + call_routes_outside_the_openai_api(world, None).await; +} + +#[when("the client repeats those calls with the key")] +async fn client_repeats_calls_with_key(world: &mut E2eWorld) { + call_routes_outside_the_openai_api(world, Some(PROTECTED_ENDPOINT_KEY)).await; +} + +#[then("every route outside the OpenAI API refuses the call as unauthorized")] +async fn assert_other_routes_refused(world: &mut E2eWorld) { + let others: Vec<_> = world + .route_statuses + .iter() + .filter(|(route, _)| route != HEALTH_ROUTE) + .collect(); + assert!(!others.is_empty(), "no routes were called"); + let unrefused: Vec<_> = others + .into_iter() + .filter(|(_, status)| *status != 401) + .collect(); + assert!( + unrefused.is_empty(), + "routes answered a client without the key instead of 401: {unrefused:?}" + ); +} + +#[then("the health check answers without the key")] +async fn assert_health_answers(world: &mut E2eWorld) { + let health = world + .route_statuses + .iter() + .find(|(route, _)| route == HEALTH_ROUTE) + .expect("the health check was not called"); + assert!( + (200..300).contains(&health.1), + "{HEALTH_ROUTE} answered {}; the liveness probe must stay open", + health.1 + ); +} + +#[then("every one of those calls succeeds")] +async fn assert_all_routes_succeed(world: &mut E2eWorld) { + assert!(!world.route_statuses.is_empty(), "no routes were called"); + let failed: Vec<_> = world + .route_statuses + .iter() + .filter(|(_, status)| !(200..300).contains(status)) + .collect(); + assert!( + failed.is_empty(), + "routes refused a client presenting the key: {failed:?}" + ); +} + #[when("the user serves a model on GPU from the installed runtime")] async fn user_serves_gpu_model_from_installed_runtime(world: &mut E2eWorld) { // Unlike `setup_active_runtime`, this scenario intentionally keeps the @@ -638,6 +820,12 @@ async fn user_serves_gpu_model_from_installed_runtime(world: &mut E2eWorld) { } async fn serve_gpu_model(world: &mut E2eWorld) { + serve_gpu_model_with_key(world, None).await; +} + +/// Serve the host's GPU model, protecting the endpoint with `api_key` when one +/// is given (`--require-api-key`, so even this loopback bind demands it). +async fn serve_gpu_model_with_key(world: &mut E2eWorld, api_key: Option<&str>) { // Serve by the canonical HuggingFace ID (not the `qwen2.5` alias) with an // explicit engine matching this host. This step is a *precondition* for // scenarios that test inference/chat behavior, so it must not fail for @@ -676,12 +864,24 @@ async fn serve_gpu_model(world: &mut E2eWorld) { for attempt in 1..=attempts { made = attempt; let device_state = ensure_serve_port_free().await; - let (stdout, stderr, rc) = crate::run_rocm_bounded( + let mut args = vec!["serve", model, "--engine", engine, "--managed"]; + // The key goes through `ROCM_SERVE_API_KEY`, as `rocm serve --help` + // advises, not `--api-key`: argv is recorded verbatim in the uploaded + // `results/commands.jsonl`. + let mut envs = Vec::new(); + if let Some(key) = api_key { + args.push("--require-api-key"); + envs.push(("ROCM_SERVE_API_KEY", key)); + } + let (stdout, stderr, rc) = crate::run_rocm_bounded_with_env( world, - &["serve", model, "--engine", engine, "--managed"], + &args, + &envs, e2e_cucumber::serve_log::serve_cli_budget(timeout_secs), ); - if rc == 0 && model_is_ready(&MODELS_URL, Some(ready_substr), timeout_secs).await { + if rc == 0 + && model_is_ready_with_key(&MODELS_URL, Some(ready_substr), timeout_secs, api_key).await + { world.endpoint = Some(SERVE_BASE_URL.to_string()); world.model_name = Some(model.to_string()); return; @@ -877,10 +1077,21 @@ async fn setup_aim_model(world: &mut E2eWorld) { serve_aim_canary(world).await; } +#[given("a model is being served on aim behind an API key")] +async fn setup_aim_model_behind_key(world: &mut E2eWorld) { + serve_aim_canary_with_key(world, Some(PROTECTED_ENDPOINT_KEY)).await; +} + /// Serves the same hardware-verified AIM canary [`setup_aim_model`] does. /// Factored out so the lifecycle-parity scenario can call it a second time, /// after stopping the service, without duplicating the image/profile pin. async fn serve_aim_canary(world: &mut E2eWorld) { + serve_aim_canary_with_key(world, None).await; +} + +/// [`serve_aim_canary`], protecting the endpoint with `api_key` when one is +/// given (`--require-api-key`, so even this loopback bind demands it). +async fn serve_aim_canary_with_key(world: &mut E2eWorld, api_key: Option<&str>) { // Serves via the raw AIM Docker image reference rather than the catalog // recipe added in ROCMAI-555 — this is a hardware-verified serve+inference // canary pinned to one exact image/profile, not a catalog-discovery test. @@ -898,7 +1109,7 @@ async fn serve_aim_canary(world: &mut E2eWorld) { // (not the Docker image reference) as `model` — confirmed live on Strix // Halo. let model = "Qwen/Qwen3.5-9B"; - serve_and_wait( + serve_and_wait_with_key( world, &[ "serve", @@ -911,6 +1122,7 @@ async fn serve_aim_canary(world: &mut E2eWorld) { ], model, "Qwen3.5-9B", + api_key, ) .await; } diff --git a/tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs b/tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs index 10d3fd7e4..a3c4dc638 100644 --- a/tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/symptom_file_steps.rs @@ -306,8 +306,9 @@ async fn told_how_to_end(world: &mut E2eWorld) { // The hint's own words. The end-of-input key it names is the one the When // step pressed, and the CLI finished because of it -- for the Linux key: // the scenario runs on Linux and WSL only, and the Windows key (Ctrl-Z then - // Enter) is checked by hand in docs/manual-testing.md section 13. The hint - // is wider than the terminal, so where it wraps is ignored. + // Enter) is checked by hand in docs/manual-testing.md's "Pasting An Error + // Into `rocm diagnose`" section. The hint is wider than the terminal, so + // where it wraps is ignored. assert!( squeeze(screen).contains(&squeeze( "Paste the error text, then end it with Ctrl-D at the start of a line \ diff --git a/xtask/src/ascii_output_contract.rs b/xtask/src/ascii_output_contract.rs index 18827f11d..6ccc50521 100644 --- a/xtask/src/ascii_output_contract.rs +++ b/xtask/src/ascii_output_contract.rs @@ -162,6 +162,12 @@ mod tests { "The `©` is in a `#` shell comment of the installer `rocm remote \ provision` uploads. The shell never echoes a comment.", ), + ( + "crates/rocm-engine-protocol/src/rocm_cli_vllm_auth.py", + "The `©` is in the `#` copyright comment of the auth middleware \ + module `rocm serve` writes for vLLM, native or in an AIM \ + container, to import. Python never prints a comment.", + ), ]; /// Individual literals that are out of scope, as