Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ Start a local OpenAI-compatible model server:

```
rocm serve <model> [--engine lemonade|vllm]
[--device gpu_required|gpu_preferred|cpu_only]
[--device gpu_required|gpu_preferred]
[--gpu auto|<index>]
[--runtime-id KEY | --env-id ID]
[--host HOST] [--port PORT]
Expand Down Expand Up @@ -356,8 +356,7 @@ not already used by another rocm-cli server (managed or foreground), falling
back to the GPU with the most free memory. Pass a single index (`--gpu 1`) to
pin a specific device. The
selected GPU is exposed to the engine via `HIP_VISIBLE_DEVICES`. Serving one
model across multiple GPUs is not supported. `--gpu` is ignored with
`--device cpu_only` (the model runs on CPU). Because selection uses the
model across multiple GPUs is not supported. Because selection uses the
`amd-smi` ordinal but is applied via `HIP_VISIBLE_DEVICES`, rocm-cli warns when
`ROCR_VISIBLE_DEVICES` is set, since the two orderings can diverge.

Expand Down
162 changes: 117 additions & 45 deletions apps/rocm/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -346,9 +346,9 @@ rocm serve qwen2.5-7b-instruct --verbose --device gpu_required")]
/// Engine to use.
#[arg(long, value_parser = SUPPORTED_ENGINES)]
engine: Option<String>,
/// Device policy [possible values: gpu_required, gpu_preferred, cpu_only].
/// Device policy.
#[arg(long)]
device: Option<String>,
device: Option<DevicePolicyArg>,
/// GPU device to serve on: `auto` (default; first free GPU) or a single
/// index like `1`.
#[arg(long, value_name = "INDEX|auto")]
Expand Down Expand Up @@ -978,6 +978,45 @@ enum TelemetryModeArg {
Off,
}

/// User-facing `rocm serve --device` policy.
///
/// Modeled as a clap `ValueEnum` so invalid input is rejected by clap's usage
/// validation (exit code 2, with the valid choices listed) rather than a generic
/// application error (exit code 1) — matching how every other enum-style CLI
/// argument behaves. The historical aliases (`auto`/`gpu` for `gpu_required`,
/// `cpu` for `cpu_only`) stay accepted for backward compatibility but are hidden
/// from the advertised list.
#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]
#[value(rename_all = "snake_case")]
enum DevicePolicyArg {
/// Require a ROCm GPU; fail if none is usable (the default).
#[value(alias = "auto", alias = "gpu")]
GpuRequired,
/// Accepted for compatibility; behaves identically to `gpu_required`
/// (rocm serve has no CPU fallback path).
GpuPreferred,
/// Accepted but always rejected: rocm serve has no CPU fallback path, so
/// this exits with an error. Hidden from `--help`, shell completions and the
/// invalid-value suggestion list so it is never advertised as usable, while
/// still parsing so the deliberate rejection message is preserved.
#[value(alias = "cpu", hide = true)]
CpuOnly,
}

impl DevicePolicyArg {
/// Canonical policy string understood by [`parse_device_policy`]. Routing the
/// parsed variant back through the string keeps `parse_device_policy` the
/// single choke point that enforces the `cpu_only` rejection across every
/// caller, rather than adding a second conversion path that could bypass it.
const fn as_policy_str(self) -> &'static str {
match self {
Self::GpuRequired => "gpu_required",
Self::GpuPreferred => "gpu_preferred",
Self::CpuOnly => "cpu_only",
}
}
}

#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]
#[value(rename_all = "snake_case")]
enum SandboxToolArg {
Expand Down Expand Up @@ -4704,7 +4743,7 @@ fn engine_recipe_enables_tool_choice(hint: Option<&EngineRecipeHint>) -> bool {
struct ServeArgs {
model: String,
engine: Option<String>,
device: Option<String>,
device: Option<DevicePolicyArg>,
gpu: Option<String>,
runtime_id: Option<String>,
env_id: Option<String>,
Expand Down Expand Up @@ -4843,7 +4882,7 @@ fn serve(args: ServeArgs) -> Result<()> {
} else {
None
};
let device_policy = parse_device_policy(device.as_deref())?;
let device_policy = parse_device_policy(device.as_ref().map(|policy| policy.as_policy_str()))?;
let gpu_selection = parse_gpu_selection(gpu.as_deref())?;
// CPU-only serving never pins a GPU, so skip GPU resolution entirely and
// surface the explicit `--gpu` as ignored rather than printing a device the
Expand Down Expand Up @@ -18053,23 +18092,7 @@ mod tests {
}
}

fn possible_values_listed_in_help(help: &str) -> Vec<String> {
let marker = "[possible values:";
let start = help
.find(marker)
.unwrap_or_else(|| panic!("help text missing `{marker}`: {help:?}"));
let rest = &help[start + marker.len()..];
let end = rest
.find(']')
.unwrap_or_else(|| panic!("unterminated possible-values list in help: {help:?}"));
rest[..end]
.split(',')
.map(|value| value.trim().to_owned())
.filter(|value| !value.is_empty())
.collect()
}

fn serve_arg_help(arg_id: &str) -> String {
fn serve_possible_values(arg_id: &str) -> Vec<String> {
let cli = Cli::command();
let serve = cli
.find_subcommand("serve")
Expand All @@ -18078,32 +18101,25 @@ mod tests {
.get_arguments()
.find(|arg| arg.get_id().as_str() == arg_id)
.unwrap_or_else(|| panic!("serve has no `{arg_id}` argument"))
.get_help()
.map(ToString::to_string)
.unwrap_or_default()
.get_possible_values()
.iter()
.map(|value| value.get_name().to_owned())
.collect()
}

// `--engine` restricts its input to `SUPPORTED_ENGINES` via a clap
// `value_parser`, so the possible values are advertised in `--help` and shell
// completion structurally (not a hand-written doc string). `--device` stays
// free-form (it accepts aliases such as `auto`/`gpu` plus the intentional
// `cpu_only` rejection), so its advertised list is hand-written and guarded
// by a sync test below. Both guards keep the advertised lists honest.
// Both `--engine` and `--device` restrict their input to a fixed set via a
// clap `value_parser`/`ValueEnum`, so invalid input is rejected as a usage
// error (exit code 2) with the accepted choices listed. Values are advertised
// in `--help` and shell completion structurally (not a hand-written doc
// string), with one deliberate exception: `--device cpu_only` is
// `#[value(hide = true)]` — still accepted so its exit-1 rejection message
// survives, but kept out of help and completion. The sync tests below keep the
// advertised lists honest; `serve_device_help_lists_match_device_policy_names`
// compares against the full `DevicePolicy` set (hidden entries included, via
// `get_possible_values`), so a dropped or renamed variant still fails.
#[test]
fn serve_engine_help_lists_match_engine_inventory() {
let cli = Cli::command();
let serve = cli
.find_subcommand("serve")
.expect("serve subcommand exists");
let engine_arg = serve
.get_arguments()
.find(|arg| arg.get_id() == "engine")
.expect("serve has an `engine` argument");
let mut listed: Vec<String> = engine_arg
.get_possible_values()
.iter()
.map(|value| value.get_name().to_owned())
.collect();
let mut listed = serve_possible_values("engine");
let mut expected: Vec<String> = builtin_engine_inventory()
.iter()
.map(|(name, _)| (*name).to_owned())
Expand All @@ -18118,7 +18134,7 @@ mod tests {

#[test]
fn serve_device_help_lists_match_device_policy_names() {
let mut listed = possible_values_listed_in_help(&serve_arg_help("device"));
let mut listed = serve_possible_values("device");
let mut expected: Vec<String> = [
DevicePolicy::GpuRequired,
DevicePolicy::GpuPreferred,
Expand All @@ -18131,7 +18147,63 @@ mod tests {
expected.sort();
assert_eq!(
listed, expected,
"serve --device help possible-values must stay in sync with DevicePolicy names"
"serve --device possible-values must stay in sync with DevicePolicy names"
);
}

#[test]
fn serve_rejects_unknown_device_policy_as_usage_error() {
// An invalid `--device` must fail clap's value validation (a usage error,
// exit code 2) rather than parsing as a free-form string and failing later
// in application logic (exit code 1). This keeps `--device` consistent with
// every other enum-style argument and lists the valid choices in the error.
let error = parse_serve(&["--device", "bogus"]).expect_err("invalid device rejected");
assert_eq!(error.kind(), clap::error::ErrorKind::InvalidValue);
}

#[test]
fn serve_accepts_device_policy_values_and_aliases() {
for (value, expected) in [
("gpu_required", DevicePolicyArg::GpuRequired),
("gpu_preferred", DevicePolicyArg::GpuPreferred),
("cpu_only", DevicePolicyArg::CpuOnly),
// Historical aliases stay accepted for backward compatibility.
("auto", DevicePolicyArg::GpuRequired),
("gpu", DevicePolicyArg::GpuRequired),
("cpu", DevicePolicyArg::CpuOnly),
] {
let cli = parse_serve(&["--device", value]).expect("device value parses");
match cli.command {
Some(Command::Serve { device, .. }) => {
assert_eq!(device, Some(expected), "device value `{value}`");
}
other => panic!("expected Serve, got {other:?}"),
}
}
}

#[test]
fn device_policy_arg_maps_through_parse_device_policy() {
// `as_policy_str` feeds the parsed variant back through
// `parse_device_policy`, the single choke point enforcing the strict
// no-CPU policy. Guard the mapping so a mis-typed arm (e.g. `CpuOnly =>
// "gpu_required"`) cannot silently turn the deliberate CPU rejection into
// a GPU-required serve while every other test stays green.
assert_eq!(
parse_device_policy(Some(DevicePolicyArg::GpuRequired.as_policy_str()))
.expect("gpu_required parses"),
DevicePolicy::GpuRequired
);
assert_eq!(
parse_device_policy(Some(DevicePolicyArg::GpuPreferred.as_policy_str()))
.expect("gpu_preferred parses"),
DevicePolicy::GpuRequired
);
// `cpu_only` must still be rejected outright rather than mapped to a
// GPU policy.
assert!(
parse_device_policy(Some(DevicePolicyArg::CpuOnly.as_policy_str())).is_err(),
"cpu_only must be rejected"
);
}

Expand Down
Loading