Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
537fd8c
feat: make model serving ports collision-safe
michaelroy-amd Aug 12, 2026
f7920a1
fix: address collision-safe serving review
michaelroy-amd Aug 18, 2026
e9fc8a1
fix(serve): keep failed-launch artifacts when a child actually spawned
juhovainio Oct 13, 2026
febc88f
fix(serve): address remaining non-blocking port-safety review notes
juhovainio Oct 13, 2026
41ad840
fix(serve): address review follow-up for port-collision safety
juhovainio Oct 15, 2026
c4f1ebc
fix(serve): gate the zombie-reaping cleanup test out on Windows
juhovainio Oct 15, 2026
741a541
fix(serve): detect an immediately-dead managed engine child on Windows
juhovainio Oct 15, 2026
7d0927c
fix(serve): unify port-collision checks and fix review findings
juhovainio Oct 15, 2026
960d0f5
fix(serve): address review feedback on port-collision and lock correc…
juhovainio Oct 15, 2026
17e931d
fix(dash-tui): flag a broken hidden host in the serve wizard summary
juhovainio Oct 15, 2026
8a5672b
fix(serve): replace non-ASCII en dash with hyphen in port range message
juhovainio Oct 16, 2026
135f960
fix(rocmd): repair restart rollback tests for write-via-rename
juhovainio Oct 16, 2026
331fb47
fix(serve): address review feedback on port-collision safety
juhovainio Oct 22, 2026
fea5e53
fix(serve): address review 5651221433 on port policy, tests and docs
juhovainio Oct 26, 2026
3fd7d53
docs(serve): add serve_port to module map; use ASCII in serve copy
juhovainio Oct 26, 2026
51902b6
Merge remote-tracking branch 'origin/main' into fix-243
juhovainio Oct 27, 2026
b4d3071
fix(serve): address review round on recovery races and test coverage
juhovainio Oct 27, 2026
4fa920d
fix(serve): refuse a differing explicit port on a live equivalent ser…
juhovainio Oct 27, 2026
f95fc20
Merge remote-tracking branch 'origin/main' into fix-243
juhovainio Oct 28, 2026
61be4b7
fix(rocmd): refuse recovery over a pending stop or removed record
juhovainio Oct 28, 2026
93d5353
Merge remote-tracking branch 'origin/main' into feat/serve-auto-port-…
juhovainio Oct 29, 2026
2cd8ad6
Merge remote-tracking branch 'origin/main' into fix-243
juhovainio Oct 29, 2026
a8daaee
Address review: pin range top, blocks_port, restart lock, and refusal…
juhovainio Oct 30, 2026
ebe4047
Merge remote-tracking branch 'origin/main' into fix-243
juhovainio Oct 30, 2026
0e11b83
test(e2e): order serve-32/33 after serve-31 and prefix id with serve
juhovainio Nov 2, 2026
943b897
Merge remote-tracking branch 'origin/main' into checks-243
juhovainio Nov 2, 2026
61c851d
fix(e2e): initialize new mock server fields in start_on_port after me…
juhovainio Nov 2, 2026
c666e3d
Merge origin/main into feat/serve-auto-port-advanced
juhovainio Nov 2, 2026
93c1e11
docs(serve): document restart launch lock, tidy review nits
juhovainio Nov 3, 2026
25bdc70
Merge origin/main into feat/serve-auto-port-advanced; address review …
juhovainio Nov 5, 2026
4bc1aea
Merge origin/main into feat/serve-auto-port-advanced; address review …
juhovainio Nov 5, 2026
759e1d0
fix(serve): refuse custom-host explicit ports held on an overlapping …
juhovainio Nov 5, 2026
14a62dd
Merge remote-tracking branch 'origin/main' into feedback-243
juhovainio Nov 5, 2026
726bb55
fix(serve): compare custom hosts as addresses; tighten spawn-liveness…
juhovainio Nov 5, 2026
3a1a631
test(serve): cover the remaining hosts_overlap branches
juhovainio Nov 5, 2026
0eddd06
Merge remote-tracking branch 'origin/main' into fix-243
juhovainio Nov 6, 2026
3e3281d
fix(serve): address review round on failed-launch cleanup, wizard por…
juhovainio Nov 6, 2026
44c76cf
fix(serve): never signal an unverifiable pid when failed-launch clean…
juhovainio Nov 6, 2026
8772efa
fix(serve): keep the spawn-time engine start token instead of re-capt…
juhovainio Nov 6, 2026
51fabc8
Merge origin/main into feat/serve-auto-port-advanced
juhovainio Nov 6, 2026
9586b97
Merge origin/main into feat/serve-auto-port-advanced
juhovainio Nov 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -863,6 +863,32 @@ daemon's environment, not yours, so a forwarded secret must also be set
there. `ROCM_CLI_AIM_READY_TIMEOUT_SECS` raises the 5-minute readiness wait
for a cold-start model download.

`--port` is optional on the default loopback host (`127.0.0.1`): omit it and
`rocm serve` picks the first free port in `11435`-`11535`, checking both
rocm-cli's own records and the OS. The non-interactive plan output (piped
output, or the Dash approval card) shows `Port: automatic; endpoint shown
after launch` while the launch is in flight and prints the concrete endpoint
once the port is actually leased; the interactive deployment summary table
only ever shows the final resolved endpoint. Pass `--port` explicitly to pin
one; on `127.0.0.1`, an explicit loopback port already reserved by another
managed service is rejected up front, naming that service and how to free it
(`rocm services stop <service-id> --yes`). A port held by an untracked process
is rejected too, but without naming the holder. `--host` to anything other
than the default
loopback address (`localhost`, `::1`, `0.0.0.0`, or any other address)
requires an explicit `--port`: automatic selection only probes `127.0.0.1` and
cannot promise a free port anywhere else. An explicit `--port` on such a host
is refused when a live managed service already holds that port on the same
address (`localhost` counts as both loopback addresses) or on a wildcard
listener that covers it (`0.0.0.0` for IPv4 addresses, `::` for any); the OS is not probed on such a
host, so any other clash is reported by the engine when it starts.

`rocm services restart` and `rocmd`'s automatic recovery are deliberately
stricter about hosts than that launch check: they refuse to bring a service
back on a port that any other live managed service has recorded, whatever host
that service is bound to, because a `0.0.0.0` listener also covers the loopback
address.

`--gpu-memory-utilization` (a fraction above 0.0 and up to 1.0 of each GPU's
total VRAM) and `--tool-call-parser` (for example `hermes`, `llama3_json`, or
`mistral`, which also turns on automatic tool choice) apply to vLLM only. Lower
Expand Down Expand Up @@ -891,6 +917,10 @@ different settings. If you previously started a service with `--temperature`
flags, or with different ones, stop the existing service first (`rocm
services stop <service-id> --yes`) or match the original flags.

Likewise, an explicit `--port` that differs from the port the running service
is already on is refused rather than silently ignored; omit `--port` to reuse
the running service, or stop it first to serve on the new port.

By default the server runs in the background under ROCm CLI's supervision and
prints a deployment summary: a progress indicator while it starts, then a table
with the status, the full inference endpoint, the API-qualified model name, and a
Expand Down Expand Up @@ -1114,6 +1144,14 @@ second stop can report the server stopped while that process still runs.
recorded process is gone; if the stop cannot confirm that, it says so, starts
nothing, and leaves the stop request recorded.

After the stop confirms, `restart` takes the same launch lock `rocm serve` uses
and re-checks the port before it respawns: if another live managed service is
now recorded on that port, or (on the default `127.0.0.1` host only) the port is
bound by another process, it refuses and the service stays stopped (`rocm services list` shows it). The lock wait is
bounded (5 seconds), unlike `rocm serve`'s: if a launch still holds the lock,
`restart` fails saying the server was stopped but not restarted and names
`rocm services restart <id> --yes` to run once that launch finishes.

`remove` deletes one record that is no longer running, together with its log,
its engine state file, and its endpoint key file; a running server is refused,
so stop it first. `prune` does the same in bulk, always leaves running servers
Expand Down
99 changes: 96 additions & 3 deletions apps/rocm/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -443,9 +443,16 @@ rocm serve qwen --verbose --device gpu_required")]
/// Host address to bind.
#[arg(long, default_value = DEFAULT_LOCAL_HOST)]
host: String,
/// TCP port to bind.
#[arg(long, default_value_t = rocm_core::DEFAULT_LOCAL_PORT)]
port: u16,
/// TCP port to bind. Omit it to choose an available port automatically,
/// starting at 11435, on the default 127.0.0.1 host. Any other --host
/// requires an explicit --port: proactive collision detection is claimed
/// only for local 127.0.0.1 serving.
// `allow_hyphen_values` so a negative value such as `-1` reaches
// `parse_serve_port` and gets the domain error naming the valid range,
// instead of clap rejecting it as an unknown flag. Mirrors the same
// choice made for `--gpu-memory-utilization` below.
#[arg(long, value_parser = crate::serve_port::parse_serve_port, allow_hyphen_values = true)]
port: Option<u16>,
/// Attach to the server in this terminal and stream its logs (Ctrl-D to
/// detach and leave it running, Ctrl-C to stop). Same as --verbose.
#[arg(long, conflicts_with = "managed")]
Expand Down Expand Up @@ -1765,6 +1772,21 @@ mod tests {
.collect()
}

fn serve_arg_help(arg_id: &str) -> String {
let mut cli = Cli::command();
let serve = cli
.find_subcommand_mut("serve")
.expect("serve subcommand exists");
let arg = serve
.get_arguments()
.find(|arg| arg.get_id().as_str() == arg_id)
.unwrap_or_else(|| panic!("serve has no `{arg_id}` argument"));
arg.get_long_help()
.map(ToString::to_string)
.or_else(|| arg.get_help().map(ToString::to_string))
.unwrap_or_default()
}

// Both `--engine` and `--device` restrict their input to a fixed set via a
// clap `value_parser`/`ValueEnum`, so invalid input is rejected as a usage
// error (exit code 2) with the accepted choices listed. Values are advertised
Expand Down Expand Up @@ -1963,6 +1985,77 @@ mod tests {
}
}

#[test]
fn serve_without_port_flag_requests_automatic_selection() {
let cli = parse_serve(&[]).expect("bare serve parses");
match cli.command {
Some(Command::Serve { port, host, .. }) => {
assert_eq!(port, None, "omitted --port must stay unresolved");
assert_eq!(host, DEFAULT_LOCAL_HOST);
assert!(crate::serve_port::PortRequest::from_flag(port).is_auto());
}
other => panic!("expected Serve, got {other:?}"),
}
}

#[test]
fn serve_with_explicit_port_keeps_it_concrete() {
let cli = parse_serve(&["--port", "8000"]).expect("explicit port parses");
match cli.command {
Some(Command::Serve { port, .. }) => {
assert_eq!(port, Some(8000));
assert_eq!(
crate::serve_port::PortRequest::from_flag(port),
crate::serve_port::PortRequest::Explicit(8000)
);
}
other => panic!("expected Serve, got {other:?}"),
}
}

#[test]
fn serve_rejects_a_zero_or_out_of_range_port() {
for bad in ["0", "65536", "-1", "auto"] {
let error = parse_serve(&["--port", bad]).expect_err("must be rejected");
assert!(
error.to_string().contains("between 1 and 65535"),
"unexpected error for --port {bad}: {error}"
);
}
}

#[test]
fn hidden_engine_serve_command_keeps_a_concrete_default_port() {
// Only the user-facing verb gained automatic selection; the direct engine
// entry point still requires (and defaults to) a concrete port.
let cli = Cli::try_parse_from([
"rocm",
"__engine-serve-http",
"lemonade",
"svc-1",
"qwen",
"--state-path",
"state.json",
])
.expect("hidden engine command parses");
match cli.command {
Some(Command::EngineServeHttp { port, host, .. }) => {
assert_eq!(port, rocm_core::DEFAULT_LOCAL_PORT);
assert_eq!(host, DEFAULT_LOCAL_HOST);
}
other => panic!("expected EngineServeHttp, got {other:?}"),
}
}

#[test]
fn serve_port_help_documents_omission_and_the_custom_host_rule() {
let help = serve_arg_help("port");
assert!(help.contains("11435"), "{help}");
assert!(help.contains("automatically"), "{help}");
assert!(help.contains("127.0.0.1"), "{help}");
assert!(help.contains("explicit --port"), "{help}");
}

#[test]
fn serve_verbose_conflicts_with_managed() {
// `--verbose` streams logs in the foreground; a backgrounded managed
Expand Down
1 change: 1 addition & 0 deletions apps/rocm/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ mod runtime_services;
mod runtime_uninstall;
mod runtimes_cmd;
mod serve_cmd;
mod serve_port;
mod serve_recipe;
mod serve_summary;
mod service_control;
Expand Down
Loading
Loading