Skip to content
6 changes: 5 additions & 1 deletion apps/rocm/src/therock/resolve/wheel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1927,7 +1927,11 @@ mod tests {
);
assert_eq!(nightly[1], "torch[device-gfx942]==2.13.*");

let no_row = published_pip_requirements("10.2.0", "2.12.0+rocm10.2.0", "gfx942", false);
let ten_two = published_pip_requirements("10.2.0", "2.11.0.3", "gfx942", false);
assert_eq!(ten_two[1], "torch[device-gfx942]==2.14.*");
assert_eq!(ten_two[2], "torchvision[device-gfx942]==0.29.*");

let no_row = published_pip_requirements("10.3.0", "2.12.0+rocm10.3.0", "gfx942", false);
assert_eq!(no_row[1], "torch[device-gfx942]");
assert_eq!(no_row[2], "torchvision[device-gfx942]");
}
Expand Down
7 changes: 7 additions & 0 deletions docs/release-trust.md
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,13 @@ Set the gate and the specific base variable together, and only in tests or
deliberate manual QA against a fixture server. The `therock-next` E2E scenarios
use exactly this pairing to exercise next-layout dispatch hermetically.

A wheel install records the pip index it resolved from, including an overridden
base, in the runtime's manifest as `index_url`. `rocm engines install vllm` reads
that value back for a ROCm 10.2 runtime and fetches the SDK's own torch stack from
it, without checking its scheme or host, so an override used for one install keeps
being used for that runtime's later vLLM installs, after the override is unset.
Remove and reinstall the runtime, or edit its manifest, to stop that.

## ComfyUI Source Archive Override

rocm-cli hardcodes the URL it downloads the ComfyUI source archive from. It
Expand Down
36 changes: 27 additions & 9 deletions docs/vllm.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,15 +109,33 @@ SDK and torch wheels are published for 3.10 to 3.14, and `rocm install sdk
vLLM install then refuses that runtime; re-running the install with
`--python-version 3.14` moves it back.

This route is selected by `major.minor`, so `10.0.0` and `10.1.0` discover
through *different* rows: they pin different vLLM minors and read vLLM from
different index URLs, because AMD stages ROCm 10.1's vLLM builds on a separate
host from 10.0's production index. Both rows take their torch stack from the
same `whl-next` index, which is also where `rocm install sdk` resolves either
SDK's own `+rocmX.Y` torch from. vLLM's discovery is independent of that: it
resolves its own torch, torchvision, and torchaudio pins fresh from the row's
own static version prefixes, instead of reusing what the SDK install resolved.
The SDK install might have resolved a different torch version than the row pins.
This route is selected by `major.minor`, so `10.0.0`, `10.1.0` and `10.2.0`
discover through *different* rows, each pinning its own vLLM minor. `10.0.0` reads
vLLM from the production index; `10.1.0` and `10.2.0` read it from a separate
staging host, where AMD stages their vLLM builds. The `10.0.0` and `10.1.0` rows
take their torch stack from the stable `whl-next` index, which is also where
`rocm install sdk` resolves those SDKs' own `+rocmX.Y` torch from. For those two
rows vLLM's discovery is independent of the SDK install: it resolves its own
torch, torchvision, and torchaudio pins fresh from the row's own static version
prefixes, instead of reusing what the SDK install resolved. The SDK install might
have resolved a different torch version than the row pins.

The `10.2.0` row does not discover its torch stack: it keeps the torch,
torchvision and torchaudio the SDK install already resolved (`torch 2.14.*`,
torchvision `0.29.*`, torchaudio `2.11.*`, all `+rocm10.2.0aDATE`), checks them
against the row's series and the SDK's ROCm line, and realigns to those exact
builds from the index the SDK was installed from (the one its runtime manifest
records, so a nightly SDK uses the nightly index and a release SDK the release
one; the row's nightly index when no manifest for the SDK's interpreter recorded
one or the registry could not be read, and the install output says which). A series match
would not work there: the nightly index serves 10.2 and 10.3 builds side by side and the newest `torch 2.14.*` is a
`+rocm10.3` build. The staging host's `+rocm10.2.0rcN` torch is not used: it
requires `rocm==10.2.0rc2` and `amd-torch-device-gfx*==...rc2` payloads that
staging does not serve. The row has been installed and imported (`import
vllm._C`) against a 10.2 SDK. A GPU serve has been run on WSL2 only with the
runtime linked to its own `librocdxg.so`, never against a release SDK, and the 10.2 vLLM
wheel is published for `linux_x86_64` only. The nightly channel resolves to
ROCm 10.3, which has no vLLM build and no row.

One value flows the other way. A row's exact torchaudio pin, where it has one
instead of a series, is also the hardware-verified build `rocm install sdk` pins
Expand Down
100 changes: 87 additions & 13 deletions engines/vllm/src/discover_builds.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,20 @@
/// `runtime.rs` and `wheel_builds.rs` can both match against it without either
/// importing the other's discover-build code.
pub(crate) fn rocm_sdk_series_matches(recorded: &str, table_key: &str) -> bool {
fn series(version: &str) -> Option<(u64, u64)> {
let mut parts = version.trim().split('.');
let major = parts.next()?.parse().ok()?;
let minor: String = parts
.next()?
.chars()
.take_while(char::is_ascii_digit)
.collect();
Some((major, minor.parse().ok()?))
}
series(recorded).is_some_and(|version| Some(version) == series(table_key))
rocm_series(recorded).is_some_and(|version| Some(version) == rocm_series(table_key))
}

/// The `(major, minor)` of a ROCm version string, ignoring patch and any
/// dev/pre-release suffix.
pub(crate) fn rocm_series(version: &str) -> Option<(u64, u64)> {
let mut parts = version.trim().split('.');
let major = parts.next()?.parse().ok()?;
let minor: String = parts
.next()?
.chars()
.take_while(char::is_ascii_digit)
.collect();
Some((major, minor.parse().ok()?))
}

/// How a [`VllmRocmDiscoverBuild`] row constrains the release discovery may
Expand Down Expand Up @@ -47,10 +50,26 @@ impl DiscoverVersion {
}
}

/// Where a [`VllmRocmDiscoverBuild`] row gets its torch, torchvision and
/// torchaudio from.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TorchStack {
/// Resolved from the row's `torch_index_url` by series, newest first.
Discover,
/// The versions the SDK install already put in the runtime, checked
/// against the row's series and ROCm line. For an index that serves
/// several ROCm lines at once, where the newest build of a series belongs
/// to the newest line: the nightly index lists about two dozen
/// `+rocm10.3.0aDATE` torch builds after the last `+rocm10.2.0a20261022`
/// one, and PEP 440 has no local-version wildcard to ask for the line.
FromInstalledSdk,
}

/// A ROCm SDK version whose vLLM/flash-attn/amd-aiter wheels aren't published
/// under a fixed filename (AMD rotates the dev-tag suffix constantly), so the
/// exact wheel must be discovered from the index at install time instead of
/// pinned in `wheel_builds::VLLM_ROCM_BUILD_TABLE`.
#[derive(Clone, Copy)]
pub(crate) struct VllmRocmDiscoverBuild {
pub(crate) rocm_sdk_version: &'static str,
/// PEP 425 interpreter tag this row's wheels are published for. AMD builds
Expand Down Expand Up @@ -87,6 +106,7 @@ pub(crate) struct VllmRocmDiscoverBuild {
/// [`verified_torchaudio_release`](crate::wheel_builds::verified_torchaudio_release): set one only for a build confirmed to
/// load against the row's torch on real hardware, never as a guess.
pub(crate) torchaudio_version: DiscoverVersion,
pub(crate) torch_stack: TorchStack,
}

/// Every ROCm SDK `major.minor` line the engine installs by live wheel
Expand Down Expand Up @@ -120,6 +140,7 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[
// establish, and this row has no coverage on the self-hosted lanes.
// `rocm install sdk` reads this field too: see `torchaudio_version`.
torchaudio_version: DiscoverVersion::Series("2.11"),
torch_stack: TorchStack::Discover,
},
VllmRocmDiscoverBuild {
rocm_sdk_version: "10.1.0",
Expand Down Expand Up @@ -155,6 +176,39 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[
// confirmed-working builds instead. `rocm install sdk` pins this same
// build for the 10.1 line: see `torchaudio_version`.
torchaudio_version: DiscoverVersion::Exact("2.11.0.2"),
torch_stack: TorchStack::Discover,
},
VllmRocmDiscoverBuild {
rocm_sdk_version: "10.2.0",
python_tag: "cp314",
// Same staging host as 10.1: it publishes the `.rocm102` vLLM builds
// (`0.32.1.dev0+g<sha>.d<date>.rocm102`) next to the 10.1 ones.
vllm_index_url: "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/vllm/",
// The torch stack is the SDK's own, fetched again from the index its
// runtime manifest records (see `TorchStack::FromInstalledSdk`); the
// nightly index below is only the fallback for a manifest that
// recorded none. Not the staging
// host's `+rocm10.2.0rcN` torch: that build requires
// `amd-torch-device-gfx*==2.14.0+rocm10.2.0rcN` and
// `rocm==10.2.0rc2`, which staging answers 403 for, so a GPU payload
// for it cannot be installed, and it is a different release than the
// `10.2.0aDATE` SDK that `rocm install sdk` installs. The 10.2 vLLM
// wheel imports (`import vllm._C`) against the SDK's own torch
// `2.14.0+rocm10.2.0a20261022`.
torch_index_url: "https://nightly.repo.amd.com/rocm/whl-next/",
vllm_version_prefix: "0.32",
flash_attn_version_prefix: "2.8",
amd_aiter_version_prefix: "0.1",
// 2.14 or newer, not 2.13: with `torch 2.13.0+rocm10.1.0`,
// `import vllm._C` fails on the undefined symbol
// `_ZN3c104cuda19CUDAErrorLogCaptureC1Ev`; 2.14 imports cleanly.
torch_version_prefix: "2.14",
torchvision_version_prefix: "0.29",
// A series match, not Exact: an Exact pin would make `rocm install
// sdk` treat it as verified for a 10.2 SDK, which nobody has run on a
// GPU.
torchaudio_version: DiscoverVersion::Series("2.11"),
torch_stack: TorchStack::FromInstalledSdk,
},
];

Expand Down Expand Up @@ -198,7 +252,7 @@ mod tests {
.collect();
assert_eq!(
versions,
["10.0.0", "10.1.0"],
["10.0.0", "10.1.0", "10.2.0"],
"the discover-build table's versions or order changed: install routing, the \
launch environment and the fail-closed static fallback all follow it"
);
Expand Down Expand Up @@ -277,8 +331,28 @@ mod tests {
DiscoverVersion::Exact("2.11.0.2")
);

let ten_two = vllm_rocm_discover_build("10.2.0a20261022").expect("10.2 has a discover row");
assert_eq!(
ten_two.vllm_index_url,
"https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/vllm/"
);
// Still read by the torch install and the realign for the 10.2 row, as the
// fallback when the runtime manifest records no index of its own.
assert_eq!(
ten_two.torch_index_url,
"https://nightly.repo.amd.com/rocm/whl-next/"
);
assert_eq!(ten_two.vllm_version_prefix, "0.32");
// 2.14, not 2.13: vLLM 0.32's `_C` needs a torch 2.14 symbol.
assert_eq!(ten_two.torch_version_prefix, "2.14");
assert_eq!(ten_two.torchvision_version_prefix, "0.29");
// The nightly index serves 10.2 and 10.3 torch side by side, so the
// 10.2 row reuses the SDK's own stack; the other rows discover theirs.
assert_eq!(ten_two.torch_stack, TorchStack::FromInstalledSdk);
assert_eq!(ten_one.torch_stack, TorchStack::Discover);

// An unpublished line must not borrow another line's wheels.
assert!(vllm_rocm_discover_build("10.2.0").is_none());
assert!(vllm_rocm_discover_build("10.3.0").is_none());
assert!(vllm_rocm_discover_build("7.2.3").is_none());
}
}
Loading
Loading