Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 36 additions & 19 deletions apps/rocm/src/serve_cmd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,8 @@ use crate::service_launch::{
drop_orphaned_endpoint_key_on_already_running, e2e_simulate_oom_launch,
ensure_background_helper_running_quiet, ensure_public_bind_engine_supported,
managed_service_reuse_refusal, no_usable_gpu_refusal, print_managed_launch_plain,
resolve_endpoint_auth, run_attached_service, start_managed_service, validate_bind_host,
wait_begin_client_config,
resolve_endpoint_auth, run_attached_service, scripted_managed_engine_startup_failure,
start_managed_service, validate_bind_host, wait_begin_client_config,
};
use crate::{
GatedEngineAction, SandboxToolArg, cli_progress, device_policy_name,
Expand Down Expand Up @@ -421,12 +421,15 @@ fn aim_board_note(
/// AMD GPU.
///
/// Not "exactly when the bail fires", and the difference is only in test-hook
/// builds: the bail also requires `!scripted_backend_failure` (see the `if`
/// below it), which is `cfg!(feature = "e2e-test-hooks")`/env-gated. Under that
/// scenario the gate can open while the bail cannot fire. That is deliberate —
/// the hook stands in for the GPU the lane does not have, so the reuse path it
/// exercises must still behave as it would on a real GPU-less host — and it
/// costs nothing in a shipped binary, where the term can never be true.
/// builds: the bail is also waived by `gpu_bail_waived_by_test_hook` in
/// `serve()` — the scripted Lemonade backend failure or free-space override,
/// the simulated OOM launch, and the scripted engine startup death — each
/// compiled in only with a test feature and armed by an environment variable.
/// Under any of them the gate can open while the bail cannot fire. That is
/// deliberate — the hook stands in for the GPU the lane does not have, so the
/// reuse path it exercises must still behave as it would on a real GPU-less
/// host — and it costs nothing in a shipped binary, where none of the terms can
/// be true.
///
/// `host_has_no_usable_gpu` is therefore the bail's own predicate, computed once
/// by the caller and handed to both, so the two cannot drift apart. They did:
Expand Down Expand Up @@ -767,7 +770,12 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> {
|| selection.env_id.is_some()
|| engine_manages_own_runtime(&selected_engine);
if can_resolve_model {
if engine_manages_own_runtime(&selected_engine) {
// Not under the scripted startup death, as for the same call below:
// that scenario asserts the launch, and a real runtime download is
// exactly what its ungated lane cannot do.
if engine_manages_own_runtime(&selected_engine)
&& !scripted_managed_engine_startup_failure()
{
ensure_self_managed_engine_ready(&paths, &mut config, &selected_engine)?;
}
let probe = engine_request::<_, ResolveModelResponse>(
Expand Down Expand Up @@ -877,12 +885,16 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> {
]
.iter()
.any(|variable| std::env::var_os(variable).is_some());
if !cpu_only
&& !scripted_backend_failure
&& !e2e_simulate_oom_launch()
&& !reuse_existing
&& host_has_no_usable_gpu
{
// The scripted startup-death scenario bypasses the same host precondition, for
// the same reason: it asserts what happens *after* a spawn, so it must reach
// one without real GPU hardware.
//
// Every test hook that waives this bail is named here once, so the bail and
// `gpu_refusal_waived_for_reuse` below cannot disagree about which ones do.
let gpu_bail_waived_by_test_hook = scripted_backend_failure
|| e2e_simulate_oom_launch()
|| scripted_managed_engine_startup_failure();
if !cpu_only && !gpu_bail_waived_by_test_hook && !reuse_existing && host_has_no_usable_gpu {
bail!("{}", no_usable_gpu_refusal(&device_policy));
}
// The Docker probe comes after every refusal that needs no daemon (the AIM
Expand All @@ -891,11 +903,11 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> {
ensure_engine_allowed_on_platform(&selected_engine, GatedEngineAction::Serve)?;
// Whether the refusal above was waived *only* because a live service was
// expected to be reused. `reuse_existing` already implies `!cpu_only` and
// `host_has_no_usable_gpu` (the pre-gate requires both), so the scripted
// backend failure is the one remaining term. Handed to the spawn, which
// `host_has_no_usable_gpu` (the pre-gate requires both), so the test-hook
// waiver is the one remaining term. Handed to the spawn, which
// re-checks reuse authoritatively under the launch lock and raises the
// refusal itself if the service is gone by then.
let gpu_refusal_waived_for_reuse = reuse_existing && !scripted_backend_failure;
let gpu_refusal_waived_for_reuse = reuse_existing && !gpu_bail_waived_by_test_hook;
// `--gpu` selects by the amd-smi `gpu` ordinal but is exported via
// `HIP_VISIBLE_DEVICES`; those orderings can diverge when
// `ROCR_VISIBLE_DEVICES`/partitioning is in play, so warn at serve time.
Expand Down Expand Up @@ -996,9 +1008,13 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> {
device_policy_name(&device_policy)
);
}
// Preparation is skipped for the scripted startup-death scenario too: it
// asserts the launch, not the install, and a real runtime download is exactly
// what the ungated lane cannot do.
if !matches!(device_policy, DevicePolicy::CpuOnly)
&& engine_manages_own_runtime(&selected_engine)
&& resolved_model.is_none()
&& !scripted_managed_engine_startup_failure()
{
ensure_self_managed_engine_ready(&paths, &mut config, &selected_engine)?;
}
Expand Down Expand Up @@ -1165,7 +1181,8 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> {
spinner.borrow_mut().clear();
eprintln!(
"Waiting for {waiting_on}; Ctrl-C stops waiting, the server keeps running. \
Follow: rocm logs --service {waiting_on} --follow"
Follow: {}",
crate::service_hints::service_logs_follow_hint(waiting_on)
);
if let Some(config) = wait_begin_client_config(
waiting_on,
Expand Down
10 changes: 5 additions & 5 deletions apps/rocm/src/serve_summary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String {
),
(
"logs",
format!("rocm logs --service {}", summary.service_id),
crate::service_hints::service_logs_hint(&summary.service_id),
),
],
]
Expand Down Expand Up @@ -266,8 +266,8 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String {
};
let _ = writeln!(
out,
" note: {explanation} - follow `rocm logs --service {} --follow` or check `rocm services list`",
summary.service_id
" note: {explanation} - follow `{}` or check `rocm services list`",
crate::service_hints::service_logs_follow_hint(&summary.service_id)
);
}
if let (true, Some(reason)) = (smoke_failed, summary.smoke_failure.as_deref()) {
Expand All @@ -276,8 +276,8 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String {
// empty, hedged: the reason may be rocm's own plumbing, not the engine.
let _ = writeln!(
out,
" note: the smoke test could not complete, so no metrics were measured ({reason}); the server may not be able to serve requests - check `rocm logs --service {id}`",
id = summary.service_id
" note: the smoke test could not complete, so no metrics were measured ({reason}); the server may not be able to serve requests - check `{logs}`",
logs = crate::service_hints::service_logs_hint(&summary.service_id)
);
}
for note in &summary.notes {
Expand Down
40 changes: 39 additions & 1 deletion apps/rocm/src/service_hints.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
//
// SPDX-License-Identifier: MIT

//! The one place every `rocm services <action> <id> --yes` hint is built.
//! The one place every `rocm services <action> <id> --yes` hint is built, and
//! the `rocm logs --service <id>` hints beside them.
//!
//! `main.rs`, `remote/mod.rs`, and `serve_summary.rs` each used to format this
//! string by hand, and it drifted: `serve_summary.rs`'s stop row printed the
Expand Down Expand Up @@ -48,6 +49,21 @@ pub(crate) fn service_forced_remove_hint(service_id: &str) -> String {
)
}

/// The directly-runnable `rocm logs --service <id>` hint, which prints a local
/// managed service's own log, for a message that sends the user there to see
/// why the server exited.
pub(crate) fn service_logs_hint(service_id: &str) -> String {
format!("rocm logs --service {}", shell_quote(service_id))
}

/// The directly-runnable `rocm logs --service <id> --follow` hint, which keeps
/// printing a local managed service's log as it grows.
pub(crate) fn service_logs_follow_hint(service_id: &str) -> String {
// Spelled out rather than built on `service_logs_hint`, so the advised-command
// contract test reads `--follow` too.
format!("rocm logs --service {} --follow", shell_quote(service_id))
}

/// The directly-runnable `rocm services stop <id> --yes` hint.
pub(crate) fn service_stop_hint(service_id: &str) -> String {
service_action_hint("stop", service_id)
Expand Down Expand Up @@ -145,5 +161,27 @@ mod tests {
);
}
}

// Mutation: printing `service_id` bare in either logs builder fails this
// for any id carrying whitespace or a quote.
#[test]
fn the_logs_hint_denotes_the_argv_it_names(service_id in hostile_arg()) {
let argv = |follow: bool| {
let mut argv = vec![
"rocm".to_owned(),
"logs".to_owned(),
"--service".to_owned(),
service_id.clone(),
];
if follow {
argv.push("--follow".to_owned());
}
Some(argv)
};
let hint = service_logs_hint(&service_id);
prop_assert_eq!(shlex::split(&hint), argv(false), "{}", hint);
let hint = service_logs_follow_hint(&service_id);
prop_assert_eq!(shlex::split(&hint), argv(true), "{}", hint);
}
}
}
9 changes: 7 additions & 2 deletions apps/rocm/src/service_launch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,12 @@
//!
//! - `endpoint_auth`: bind-host validation and the endpoint API key checks.
//! - `launch_transaction`: the locked port allocation decision, the cleanup of a
//! failed launch, and the reuse refusals.
//! - `spawn`: the detached engine child, its process and stdio plumbing, and
//! failed launch, the reuse refusals, and the guarded read-modify-write
//! (`update_managed_record_on_disk`) every launch write after the claim goes
//! through, so a stop or removal made meanwhile is not overwritten or undone.
//! - `spawn`: the detached engine child, its process and stdio plumbing, the
//! startup check that fails a launch whose engine exits inside the settle
//! window (on Windows through the spawn's process handle), and
//! `ManagedLaunchReport`.
//! - `readiness`: waiting on a service, its liveness, and the fault-injection
//! seam (an OOM launch, or a ready launch).
Expand Down Expand Up @@ -69,6 +73,7 @@ pub(crate) use helper_autostart::{
};
pub(crate) use launch_transaction::{managed_service_reuse_refusal, no_usable_gpu_refusal};
pub(crate) use readiness::e2e_simulate_oom_launch;
pub(crate) use spawn::scripted_managed_engine_startup_failure;
#[cfg(not(windows))]
pub(crate) use spawn::{
attach_background_stdio, detach_background_command, managed_engine_startup_failure_detail,
Expand Down
Loading
Loading