From 904a53b72b2c6e09ba264c5a28cf9519cc1af20f Mon Sep 17 00:00:00 2001 From: juhovainio Date: Mon, 16 Nov 2026 20:35:35 +0200 Subject: [PATCH 1/7] feat(vllm): add a ROCm 10.2 discover row and teach the local-version guard .rocmNNN The staging index publishes vLLM 0.32 wheels for ROCm 10.2 (local version spelled .rocm102), but the discover table stopped at 10.1, so the install failed closed on a 10.2 SDK. Add the 10.2.0 row (cp314, staging index, torch 2.14, torchvision 0.29). whl-next has no +rocm10.2 torch, so the row borrows the 10.1 stack and the new torch_stack_matches_sdk flag skips the local-version check for the torch triplet on that row only. The guard now also compares the compact .rocmNNN spelling, so a wheel for another line is still rejected. Refs #1108 Signed-off-by: juhovainio --- apps/rocm/src/therock/resolve/wheel.rs | 6 +- docs/vllm.md | 15 ++++- engines/vllm/src/discover_builds.rs | 77 ++++++++++++++++++++---- engines/vllm/src/install.rs | 83 ++++++++++++++++++++++++-- engines/vllm/src/runtime.rs | 5 +- engines/vllm/src/wheel_builds.rs | 11 +++- 6 files changed, 171 insertions(+), 26 deletions(-) diff --git a/apps/rocm/src/therock/resolve/wheel.rs b/apps/rocm/src/therock/resolve/wheel.rs index 381929ba0f..33cef9ada7 100644 --- a/apps/rocm/src/therock/resolve/wheel.rs +++ b/apps/rocm/src/therock/resolve/wheel.rs @@ -1927,7 +1927,11 @@ mod tests { ); assert_eq!(nightly[1], "torch[device-gfx942]==2.13.*"); - let no_row = published_pip_requirements("10.2.0", "2.12.0+rocm10.2.0", "gfx942", false); + let ten_two = published_pip_requirements("10.2.0", "2.11.0.3", "gfx942", false); + assert_eq!(ten_two[1], "torch[device-gfx942]==2.14.*"); + assert_eq!(ten_two[2], "torchvision[device-gfx942]==0.29.*"); + + let no_row = published_pip_requirements("10.3.0", "2.12.0+rocm10.3.0", "gfx942", false); assert_eq!(no_row[1], "torch[device-gfx942]"); assert_eq!(no_row[2], "torchvision[device-gfx942]"); } diff --git a/docs/vllm.md b/docs/vllm.md index dc5cf8c1f9..d279907c83 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -109,8 +109,8 @@ SDK and torch wheels are published for 3.10 to 3.14, and `rocm install sdk vLLM install then refuses that runtime; re-running the install with `--python-version 3.14` moves it back. -This route is selected by `major.minor`, so `10.0.0` and `10.1.0` discover -through *different* rows: they pin different vLLM minors and read vLLM from +This route is selected by `major.minor`, so `10.0.0`, `10.1.0` and `10.2.0` +discover through *different* rows: they pin different vLLM minors and read vLLM from different index URLs, because AMD stages ROCm 10.1's vLLM builds on a separate host from 10.0's production index. Both rows take their torch stack from the same `whl-next` index, which is also where `rocm install sdk` resolves either @@ -119,6 +119,17 @@ resolves its own torch, torchvision, and torchaudio pins fresh from the row's own static version prefixes, instead of reusing what the SDK install resolved. The SDK install might have resolved a different torch version than the row pins. +The `10.2.0` row is the exception to "the SDK's own `+rocmX.Y` torch": `whl-next` +publishes no `+rocm10.2` torch, and the 10.2 vLLM wheel needs torch 2.14 or +newer, so the row installs `torch 2.14.*+rocm10.1.0` with torchvision `0.29.*` +(published as `+rocm10.0.0`) and the newest torchaudio `2.11` rebuild. That +mixes ROCm lines on purpose, so the install skips the `+rocm` local +version check for those three packages on that row only; vLLM's own wheel +(spelled `.rocm102`) is still checked against the SDK. The row has been run on +one gfx1151 WSL2 host with the 10.1 SDK, not yet on a 10.2 SDK or an Instinct +GPU, and the 10.2 vLLM wheel is published for `linux_x86_64` only. The nightly +channel resolves to ROCm 10.3, which has no vLLM build and no row. + One value flows the other way. A row's exact torchaudio pin, where it has one instead of a series, is also the hardware-verified build `rocm install sdk` pins for that ROCm line. The install reads it from this table instead of copying it, diff --git a/engines/vllm/src/discover_builds.rs b/engines/vllm/src/discover_builds.rs index 99d7a92462..e9a2804d24 100644 --- a/engines/vllm/src/discover_builds.rs +++ b/engines/vllm/src/discover_builds.rs @@ -9,17 +9,20 @@ /// `runtime.rs` and `wheel_builds.rs` can both match against it without either /// importing the other's discover-build code. pub(crate) fn rocm_sdk_series_matches(recorded: &str, table_key: &str) -> bool { - fn series(version: &str) -> Option<(u64, u64)> { - let mut parts = version.trim().split('.'); - let major = parts.next()?.parse().ok()?; - let minor: String = parts - .next()? - .chars() - .take_while(char::is_ascii_digit) - .collect(); - Some((major, minor.parse().ok()?)) - } - series(recorded).is_some_and(|version| Some(version) == series(table_key)) + rocm_series(recorded).is_some_and(|version| Some(version) == rocm_series(table_key)) +} + +/// The `(major, minor)` of a ROCm version string, ignoring patch and any +/// dev/pre-release suffix. +pub(crate) fn rocm_series(version: &str) -> Option<(u64, u64)> { + let mut parts = version.trim().split('.'); + let major = parts.next()?.parse().ok()?; + let minor: String = parts + .next()? + .chars() + .take_while(char::is_ascii_digit) + .collect(); + Some((major, minor.parse().ok()?)) } /// How a [`VllmRocmDiscoverBuild`] row constrains the release discovery may @@ -87,6 +90,12 @@ pub(crate) struct VllmRocmDiscoverBuild { /// [`verified_torchaudio_release`](crate::wheel_builds::verified_torchaudio_release): set one only for a build confirmed to /// load against the row's torch on real hardware, never as a guess. pub(crate) torchaudio_version: DiscoverVersion, + /// Whether the row's torch, torchvision and torchaudio are built for the + /// row's own ROCm line, so `ensure_rocm_local_version_matches` checks their + /// `+rocm` local versions against the SDK like the vLLM wheel's. + /// `false` only where AMD publishes no torch stack for the line and the row + /// borrows another line's: the 10.2 row, see its comment. + pub(crate) torch_stack_matches_sdk: bool, } /// Every ROCm SDK `major.minor` line the engine installs by live wheel @@ -120,6 +129,7 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // establish, and this row has no coverage on the self-hosted lanes. // `rocm install sdk` reads this field too: see `torchaudio_version`. torchaudio_version: DiscoverVersion::Series("2.11"), + torch_stack_matches_sdk: true, }, VllmRocmDiscoverBuild { rocm_sdk_version: "10.1.0", @@ -155,6 +165,35 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // confirmed-working builds instead. `rocm install sdk` pins this same // build for the 10.1 line: see `torchaudio_version`. torchaudio_version: DiscoverVersion::Exact("2.11.0.2"), + torch_stack_matches_sdk: true, + }, + VllmRocmDiscoverBuild { + rocm_sdk_version: "10.2.0", + python_tag: "cp314", + // Same staging host as 10.1: it publishes the `.rocm102` vLLM builds + // (`0.32.1.dev0+g.d.rocm102`) next to the 10.1 ones. + vllm_index_url: "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/vllm/", + torch_index_url: "https://stable.repo.amd.com/rocm/whl-next/", + vllm_version_prefix: "0.32", + flash_attn_version_prefix: "2.8", + amd_aiter_version_prefix: "0.1", + // 2.14 or newer, not 2.13: with `torch 2.13.0+rocm10.1.0`, + // `import vllm._C` fails on the undefined symbol + // `_ZN3c104cuda19CUDAErrorLogCaptureC1Ev`; 2.14 imports cleanly. + // `whl-next` publishes no `+rocm10.2` torch (it stops at + // `2.14.0+rocm10.1.0`), so this row borrows the 10.1 stack, which is + // why `torch_stack_matches_sdk` is false. + torch_version_prefix: "2.14", + // torchvision 0.29 is only published as `+rocm10.0.0`, not + // `+rocm10.1.0`; it paired with torch 2.14.0+rocm10.1.0 on the + // verification box. + torchvision_version_prefix: "0.29", + // A series match: the newest `2.11` rebuild on `whl-next` was + // `2.11.0.3+rocm10.1.0`, the one the verification box ran. Not Exact, + // because an Exact pin would make `rocm install sdk` treat it as + // verified for a 10.2 SDK, which nobody has tested. + torchaudio_version: DiscoverVersion::Series("2.11"), + torch_stack_matches_sdk: false, }, ]; @@ -198,7 +237,7 @@ mod tests { .collect(); assert_eq!( versions, - ["10.0.0", "10.1.0"], + ["10.0.0", "10.1.0", "10.2.0"], "the discover-build table's versions or order changed: install routing, the \ launch environment and the fail-closed static fallback all follow it" ); @@ -277,8 +316,20 @@ mod tests { DiscoverVersion::Exact("2.11.0.2") ); + let ten_two = vllm_rocm_discover_build("10.2.0a20261022").expect("10.2 has a discover row"); + assert_eq!( + ten_two.vllm_index_url, + "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/vllm/" + ); + assert_eq!(ten_two.vllm_version_prefix, "0.32"); + // 2.14, not 2.13: vLLM 0.32's `_C` needs a torch 2.14 symbol. + assert_eq!(ten_two.torch_version_prefix, "2.14"); + assert_eq!(ten_two.torchvision_version_prefix, "0.29"); + assert!(!ten_two.torch_stack_matches_sdk); + assert!(ten_one.torch_stack_matches_sdk); + // An unpublished line must not borrow another line's wheels. - assert!(vllm_rocm_discover_build("10.2.0").is_none()); + assert!(vllm_rocm_discover_build("10.3.0").is_none()); assert!(vllm_rocm_discover_build("7.2.3").is_none()); } } diff --git a/engines/vllm/src/install.rs b/engines/vllm/src/install.rs index 5b0e500a43..5850341dbc 100644 --- a/engines/vllm/src/install.rs +++ b/engines/vllm/src/install.rs @@ -13,7 +13,8 @@ use std::path::{Path, PathBuf}; use std::process::Command as ProcessCommand; use crate::discover_builds::{ - DiscoverVersion, VllmRocmDiscoverBuild, rocm_sdk_series_matches, vllm_rocm_discover_build, + DiscoverVersion, VllmRocmDiscoverBuild, rocm_sdk_series_matches, rocm_series, + vllm_rocm_discover_build, }; use crate::repair::{RepairAssessment, assess_runtime_repair}; use crate::runtime::{ @@ -859,10 +860,16 @@ fn ensure_rocm_local_version_matches(pin: &str, rocm_sdk_version: &str) -> Resul let Some(local) = split_local_version(version).1 else { return Ok(()); }; - let Some(local_rocm) = local.strip_prefix("rocm") else { + let matches = if let Some(local_rocm) = local.strip_prefix("rocm") { + rocm_sdk_series_matches(local_rocm, rocm_sdk_version) + } else if let Some(compact) = compact_rocm_local_segment(local) { + // `.rocm102` names 10.2 with no dot between major and minor. + rocm_series(rocm_sdk_version) + .is_some_and(|(major, minor)| compact == format!("{major}{minor}")) + } else { return Ok(()); }; - if rocm_sdk_series_matches(local_rocm, rocm_sdk_version) { + if matches { return Ok(()); } bail!( @@ -870,6 +877,30 @@ fn ensure_rocm_local_version_matches(pin: &str, rocm_sdk_version: &str) -> Resul different ROCm release; the index for this ROCm line is serving another SDK's wheels" ) } +/// Applies [`ensure_rocm_local_version_matches`] to every discovered pin, except +/// the torch triplet on a row whose `torch_stack_matches_sdk` is false: that +/// row borrows another ROCm line's torch on purpose. +fn ensure_discovered_pins_match_sdk(pins: &[String], build: &VllmRocmDiscoverBuild) -> Result<()> { + for pin in pins { + let is_torch_stack = ["torch==", "torchvision==", "torchaudio=="] + .iter() + .any(|prefix| pin.starts_with(prefix)); + if is_torch_stack && !build.torch_stack_matches_sdk { + continue; + } + ensure_rocm_local_version_matches(pin, build.rocm_sdk_version)?; + } + Ok(()) +} +/// The digits of a `rocm` segment in a dotted local version, e.g. +/// `102` for `g5df63a720.d20261110.rocm102`: the spelling the 10.2 vLLM wheels +/// use, with no dot between the ROCm major and minor. +fn compact_rocm_local_segment(local: &str) -> Option<&str> { + local + .split('.') + .filter_map(|segment| segment.strip_prefix("rocm")) + .find(|digits| !digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit())) +} /// The PEP 425 interpreter tag of `python`, e.g. `cp314`. fn python_interpreter_tag(python: &Path) -> Result { let output = ProcessCommand::new(python) @@ -1031,9 +1062,7 @@ fn install_vllm_rocm10_discover( notes.extend(amd_aiter_note); let pins = vec![torch, torchvision, torchaudio, vllm, flash_attn, amd_aiter]; - for pin in &pins { - ensure_rocm_local_version_matches(pin, build.rocm_sdk_version)?; - } + ensure_discovered_pins_match_sdk(&pins, build)?; // Two separate `uv pip install` calls, each scoped to exactly one index: // see `vllm_rocm10_discover_torch_install_args` for why combining both @@ -2121,6 +2150,48 @@ fi assert!(rendered.contains("10.1.0"), "{rendered}"); assert!(rendered.contains("rocm10.0.0"), "{rendered}"); } + /// The 10.2 vLLM wheels spell their ROCm line `.rocm102`, not + /// `+rocm10.2.0`; the guard must still compare it. + #[test] + fn rocm_local_version_guard_compares_the_compact_rocm_spelling() { + let ten_two = "vllm==0.32.1.dev0+g5df63a720.d20261110.rocm102"; + assert!(ensure_rocm_local_version_matches(ten_two, "10.2.0").is_ok()); + assert!(ensure_rocm_local_version_matches(ten_two, "10.2.0a20261022").is_ok()); + let err = ensure_rocm_local_version_matches(ten_two, "10.1.0") + .expect_err("a 10.2 wheel must not install onto a 10.1 SDK"); + assert!(format!("{err:#}").contains("rocm102"), "{err:#}"); + // And a 10.1 wheel must not slip onto a 10.2 SDK through the dotted spelling. + assert!( + ensure_rocm_local_version_matches("vllm==0.29.1.dev0+rocm10.1.0.gabc", "10.2.0") + .is_err() + ); + } + /// The 10.2 row installs the 10.1 torch stack on purpose, but only the + /// torch triplet may differ from the SDK's line. + #[test] + fn discovered_pins_guard_skips_only_the_borrowed_torch_stack() { + let ten_two = vllm_rocm_discover_build("10.2.0").expect("10.2 row"); + let ten_one = vllm_rocm_discover_build("10.1.0").expect("10.1 row"); + let torch_stack = |torch: &str| { + vec![ + format!("torch=={torch}"), + "torchvision==0.29.0+rocm10.0.0".to_owned(), + "torchaudio==2.11.0.3+rocm10.1.0".to_owned(), + "vllm==0.32.1.dev0+g5df63a720.d20261110.rocm102".to_owned(), + "flash-attn==2.8.3".to_owned(), + ] + }; + assert!( + ensure_discovered_pins_match_sdk(&torch_stack("2.14.0+rocm10.1.0"), ten_two).is_ok() + ); + // The same borrowed stack is rejected on a row that expects its own. + assert!( + ensure_discovered_pins_match_sdk(&torch_stack("2.14.0+rocm10.1.0"), ten_one).is_err() + ); + // vLLM itself is still checked on the 10.2 row. + let wrong_vllm = vec!["vllm==0.29.1.dev0+rocm10.1.0.gabc".to_owned()]; + assert!(ensure_discovered_pins_match_sdk(&wrong_vllm, ten_two).is_err()); + } /// Pins `install_vllm_rocm10_discover`'s own call sequencing, not just its /// arg-builder helpers in isolation: the remaining-deps install must use /// `--config-file` (never `--extra-index-url`, which cannot tolerate the diff --git a/engines/vllm/src/runtime.rs b/engines/vllm/src/runtime.rs index 265af61e25..66971ad161 100644 --- a/engines/vllm/src/runtime.rs +++ b/engines/vllm/src/runtime.rs @@ -988,11 +988,14 @@ mod tests { assert!(runtime_has_discover_build(&runtime_with_sdk_version(Some( "10.1.0a20260611" )))); + assert!(runtime_has_discover_build(&runtime_with_sdk_version(Some( + "10.2.0" + )))); } #[test] fn runtime_has_discover_build_rejects_a_line_with_no_discover_row() { assert!(!runtime_has_discover_build(&runtime_with_sdk_version( - Some("10.2.0") + Some("10.3.0") ))); assert!(!runtime_has_discover_build(&runtime_with_sdk_version( Some("7.2.3") diff --git a/engines/vllm/src/wheel_builds.rs b/engines/vllm/src/wheel_builds.rs index 6334eb7085..6c3464a128 100644 --- a/engines/vllm/src/wheel_builds.rs +++ b/engines/vllm/src/wheel_builds.rs @@ -332,7 +332,7 @@ mod tests { // with no row still fails closed instead of taking the 7.2.3 default // pin; other lines (and unparsable text) resolve that default. assert_eq!( - discover_readers(Some("10.2.0")), + discover_readers(Some("10.3.0")), DiscoverReaders { row: None, route: VllmInstallRoute::Static, @@ -540,6 +540,7 @@ mod tests { Some("2.11.0.2") ); assert_eq!(verified_torchaudio_release("10.0.0"), None); + // The 10.2 row's torchaudio is a series match, hardware-unverified. assert_eq!(verified_torchaudio_release("10.2.0"), None); assert_eq!(verified_torchaudio_release("7.2.3"), None); } @@ -559,7 +560,11 @@ mod tests { verified_torch_stack_series("10.0.0"), Some(("2.12", "0.27")) ); - assert_eq!(verified_torch_stack_series("10.2.0"), None); + assert_eq!( + verified_torch_stack_series("10.2.0"), + Some(("2.14", "0.29")) + ); + assert_eq!(verified_torch_stack_series("10.3.0"), None); assert_eq!(verified_torch_stack_series("7.2.3"), None); } /// An unknown 10.x line fails closed rather than silently resolving the @@ -567,7 +572,7 @@ mod tests { /// [`rocm_sdk_major_matches`] alongside the narrower row lookup. #[test] fn unknown_rocm10_line_still_fails_closed() { - let err = install_target(None, Some("10.2.0")).expect_err("10.2.0 has no vLLM build"); + let err = install_target(None, Some("10.3.0")).expect_err("10.3.0 has no vLLM build"); let rendered = format!("{err:#}"); assert!( !rendered.contains("7.2.3"), From e0d25ed65b89ea636c7517344647160438e2fe0d Mon Sep 17 00:00:00 2001 From: juhovainio Date: Mon, 16 Nov 2026 20:52:49 +0200 Subject: [PATCH 2/7] fix(vllm): read the 10.2 torch stack from the staging index The staging host publishes torch 2.14.0, torchvision 0.29.2 and torchaudio 2.11.1 built for +rocm10.2.0rcN, so the 10.2 row no longer borrows the 10.1 stack. Drop the torch_stack_matches_sdk flag and the check skip it needed. Refs #1108 Signed-off-by: juhovainio --- docs/vllm.md | 16 ++++------ engines/vllm/src/discover_builds.rs | 35 +++++++++------------- engines/vllm/src/install.rs | 45 ++--------------------------- 3 files changed, 22 insertions(+), 74 deletions(-) diff --git a/docs/vllm.md b/docs/vllm.md index d279907c83..bcb5a66601 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -119,16 +119,12 @@ resolves its own torch, torchvision, and torchaudio pins fresh from the row's own static version prefixes, instead of reusing what the SDK install resolved. The SDK install might have resolved a different torch version than the row pins. -The `10.2.0` row is the exception to "the SDK's own `+rocmX.Y` torch": `whl-next` -publishes no `+rocm10.2` torch, and the 10.2 vLLM wheel needs torch 2.14 or -newer, so the row installs `torch 2.14.*+rocm10.1.0` with torchvision `0.29.*` -(published as `+rocm10.0.0`) and the newest torchaudio `2.11` rebuild. That -mixes ROCm lines on purpose, so the install skips the `+rocm` local -version check for those three packages on that row only; vLLM's own wheel -(spelled `.rocm102`) is still checked against the SDK. The row has been run on -one gfx1151 WSL2 host with the 10.1 SDK, not yet on a 10.2 SDK or an Instinct -GPU, and the 10.2 vLLM wheel is published for `linux_x86_64` only. The nightly -channel resolves to ROCm 10.3, which has no vLLM build and no row. +The `10.2.0` row reads its torch stack from the staging host too, not +`whl-next`, which publishes no `+rocm10.2` torch: `torch 2.14.*`, torchvision +`0.29.*` and the newest torchaudio `2.11` build, each `+rocm10.2.0rcN`. The row +has not yet been run on a 10.2 SDK, and the 10.2 vLLM wheel is published for +`linux_x86_64` only. The nightly channel resolves to ROCm 10.3, which has no +vLLM build and no row. One value flows the other way. A row's exact torchaudio pin, where it has one instead of a series, is also the hardware-verified build `rocm install sdk` pins diff --git a/engines/vllm/src/discover_builds.rs b/engines/vllm/src/discover_builds.rs index e9a2804d24..637b8f0c96 100644 --- a/engines/vllm/src/discover_builds.rs +++ b/engines/vllm/src/discover_builds.rs @@ -90,12 +90,6 @@ pub(crate) struct VllmRocmDiscoverBuild { /// [`verified_torchaudio_release`](crate::wheel_builds::verified_torchaudio_release): set one only for a build confirmed to /// load against the row's torch on real hardware, never as a guess. pub(crate) torchaudio_version: DiscoverVersion, - /// Whether the row's torch, torchvision and torchaudio are built for the - /// row's own ROCm line, so `ensure_rocm_local_version_matches` checks their - /// `+rocm` local versions against the SDK like the vLLM wheel's. - /// `false` only where AMD publishes no torch stack for the line and the row - /// borrows another line's: the 10.2 row, see its comment. - pub(crate) torch_stack_matches_sdk: bool, } /// Every ROCm SDK `major.minor` line the engine installs by live wheel @@ -129,7 +123,6 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // establish, and this row has no coverage on the self-hosted lanes. // `rocm install sdk` reads this field too: see `torchaudio_version`. torchaudio_version: DiscoverVersion::Series("2.11"), - torch_stack_matches_sdk: true, }, VllmRocmDiscoverBuild { rocm_sdk_version: "10.1.0", @@ -165,7 +158,6 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // confirmed-working builds instead. `rocm install sdk` pins this same // build for the 10.1 line: see `torchaudio_version`. torchaudio_version: DiscoverVersion::Exact("2.11.0.2"), - torch_stack_matches_sdk: true, }, VllmRocmDiscoverBuild { rocm_sdk_version: "10.2.0", @@ -173,27 +165,23 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // Same staging host as 10.1: it publishes the `.rocm102` vLLM builds // (`0.32.1.dev0+g.d.rocm102`) next to the 10.1 ones. vllm_index_url: "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/vllm/", - torch_index_url: "https://stable.repo.amd.com/rocm/whl-next/", + // Torch stack from the same staging host as vLLM, not `whl-next`: + // `whl-next` publishes no `+rocm10.2` torch, while staging carries a + // full 10.2 stack (`torch 2.14.0`, `torchvision 0.29.2` and + // `torchaudio 2.11.1`, each `+rocm10.2.0rcN`). + torch_index_url: "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/", vllm_version_prefix: "0.32", flash_attn_version_prefix: "2.8", amd_aiter_version_prefix: "0.1", // 2.14 or newer, not 2.13: with `torch 2.13.0+rocm10.1.0`, // `import vllm._C` fails on the undefined symbol // `_ZN3c104cuda19CUDAErrorLogCaptureC1Ev`; 2.14 imports cleanly. - // `whl-next` publishes no `+rocm10.2` torch (it stops at - // `2.14.0+rocm10.1.0`), so this row borrows the 10.1 stack, which is - // why `torch_stack_matches_sdk` is false. torch_version_prefix: "2.14", - // torchvision 0.29 is only published as `+rocm10.0.0`, not - // `+rocm10.1.0`; it paired with torch 2.14.0+rocm10.1.0 on the - // verification box. torchvision_version_prefix: "0.29", - // A series match: the newest `2.11` rebuild on `whl-next` was - // `2.11.0.3+rocm10.1.0`, the one the verification box ran. Not Exact, - // because an Exact pin would make `rocm install sdk` treat it as - // verified for a 10.2 SDK, which nobody has tested. + // A series match, which resolves the newest `2.11` build, `2.11.1`. + // Not Exact, because an Exact pin would make `rocm install sdk` treat + // it as verified for a 10.2 SDK, which nobody has tested. torchaudio_version: DiscoverVersion::Series("2.11"), - torch_stack_matches_sdk: false, }, ]; @@ -325,8 +313,11 @@ mod tests { // 2.14, not 2.13: vLLM 0.32's `_C` needs a torch 2.14 symbol. assert_eq!(ten_two.torch_version_prefix, "2.14"); assert_eq!(ten_two.torchvision_version_prefix, "0.29"); - assert!(!ten_two.torch_stack_matches_sdk); - assert!(ten_one.torch_stack_matches_sdk); + // The 10.2 torch stack is published on the staging host, unlike 10.1's. + assert_eq!( + ten_two.torch_index_url, + "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/" + ); // An unpublished line must not borrow another line's wheels. assert!(vllm_rocm_discover_build("10.3.0").is_none()); diff --git a/engines/vllm/src/install.rs b/engines/vllm/src/install.rs index 5850341dbc..ecd1ffc377 100644 --- a/engines/vllm/src/install.rs +++ b/engines/vllm/src/install.rs @@ -877,21 +877,6 @@ fn ensure_rocm_local_version_matches(pin: &str, rocm_sdk_version: &str) -> Resul different ROCm release; the index for this ROCm line is serving another SDK's wheels" ) } -/// Applies [`ensure_rocm_local_version_matches`] to every discovered pin, except -/// the torch triplet on a row whose `torch_stack_matches_sdk` is false: that -/// row borrows another ROCm line's torch on purpose. -fn ensure_discovered_pins_match_sdk(pins: &[String], build: &VllmRocmDiscoverBuild) -> Result<()> { - for pin in pins { - let is_torch_stack = ["torch==", "torchvision==", "torchaudio=="] - .iter() - .any(|prefix| pin.starts_with(prefix)); - if is_torch_stack && !build.torch_stack_matches_sdk { - continue; - } - ensure_rocm_local_version_matches(pin, build.rocm_sdk_version)?; - } - Ok(()) -} /// The digits of a `rocm` segment in a dotted local version, e.g. /// `102` for `g5df63a720.d20261110.rocm102`: the spelling the 10.2 vLLM wheels /// use, with no dot between the ROCm major and minor. @@ -1062,7 +1047,9 @@ fn install_vllm_rocm10_discover( notes.extend(amd_aiter_note); let pins = vec![torch, torchvision, torchaudio, vllm, flash_attn, amd_aiter]; - ensure_discovered_pins_match_sdk(&pins, build)?; + for pin in &pins { + ensure_rocm_local_version_matches(pin, build.rocm_sdk_version)?; + } // Two separate `uv pip install` calls, each scoped to exactly one index: // see `vllm_rocm10_discover_torch_install_args` for why combining both @@ -2166,32 +2153,6 @@ fi .is_err() ); } - /// The 10.2 row installs the 10.1 torch stack on purpose, but only the - /// torch triplet may differ from the SDK's line. - #[test] - fn discovered_pins_guard_skips_only_the_borrowed_torch_stack() { - let ten_two = vllm_rocm_discover_build("10.2.0").expect("10.2 row"); - let ten_one = vllm_rocm_discover_build("10.1.0").expect("10.1 row"); - let torch_stack = |torch: &str| { - vec![ - format!("torch=={torch}"), - "torchvision==0.29.0+rocm10.0.0".to_owned(), - "torchaudio==2.11.0.3+rocm10.1.0".to_owned(), - "vllm==0.32.1.dev0+g5df63a720.d20261110.rocm102".to_owned(), - "flash-attn==2.8.3".to_owned(), - ] - }; - assert!( - ensure_discovered_pins_match_sdk(&torch_stack("2.14.0+rocm10.1.0"), ten_two).is_ok() - ); - // The same borrowed stack is rejected on a row that expects its own. - assert!( - ensure_discovered_pins_match_sdk(&torch_stack("2.14.0+rocm10.1.0"), ten_one).is_err() - ); - // vLLM itself is still checked on the 10.2 row. - let wrong_vllm = vec!["vllm==0.29.1.dev0+rocm10.1.0.gabc".to_owned()]; - assert!(ensure_discovered_pins_match_sdk(&wrong_vllm, ten_two).is_err()); - } /// Pins `install_vllm_rocm10_discover`'s own call sequencing, not just its /// arg-builder helpers in isolation: the remaining-deps install must use /// `--config-file` (never `--extra-index-url`, which cannot tolerate the From 39c21ddc17b0e83441c2caf26d1387dc8fc703c2 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Mon, 16 Nov 2026 21:43:59 +0200 Subject: [PATCH 3/7] fix(vllm): reuse the SDK's torch stack for the 10.2 row Running the row on a real 10.2 SDK showed the staged torch does not work: the newest torch 2.14.* on the staging and nightly indexes is a +rocm10.3 build, and the staging +rocm10.2.0rcN torch needs device payloads and a rocm==10.2.0rc2 that staging does not serve. Add TorchStack::FromInstalledSdk: the row keeps the torch, torchvision and torchaudio the SDK install resolved, checks them against the row's series and ROCm line, and realigns to those exact builds from the nightly index. The other rows still discover theirs. Refs #1108 Signed-off-by: juhovainio --- docs/vllm.md | 18 ++- engines/vllm/src/discover_builds.rs | 49 ++++++--- engines/vllm/src/install.rs | 163 ++++++++++++++++++++++------ 3 files changed, 180 insertions(+), 50 deletions(-) diff --git a/docs/vllm.md b/docs/vllm.md index bcb5a66601..4e47e070e8 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -119,12 +119,18 @@ resolves its own torch, torchvision, and torchaudio pins fresh from the row's own static version prefixes, instead of reusing what the SDK install resolved. The SDK install might have resolved a different torch version than the row pins. -The `10.2.0` row reads its torch stack from the staging host too, not -`whl-next`, which publishes no `+rocm10.2` torch: `torch 2.14.*`, torchvision -`0.29.*` and the newest torchaudio `2.11` build, each `+rocm10.2.0rcN`. The row -has not yet been run on a 10.2 SDK, and the 10.2 vLLM wheel is published for -`linux_x86_64` only. The nightly channel resolves to ROCm 10.3, which has no -vLLM build and no row. +The `10.2.0` row does not discover its torch stack: it keeps the torch, +torchvision and torchaudio the SDK install already resolved (`torch 2.14.*`, +torchvision `0.29.*`, torchaudio `2.11.*`, all `+rocm10.2.0aDATE`), checks them +against the row's series and the SDK's ROCm line, and realigns to those exact +builds from the nightly index. A series match would not work there: that index +serves 10.2 and 10.3 builds side by side and the newest `torch 2.14.*` is a +`+rocm10.3` build. The staging host's `+rocm10.2.0rcN` torch is not used: it +requires `rocm==10.2.0rc2` and `amd-torch-device-gfx*==...rc2` payloads that +staging does not serve. The row has been installed and imported (`import +vllm._C`) against a 10.2 SDK, but not yet served on a GPU, and the 10.2 vLLM +wheel is published for `linux_x86_64` only. The nightly channel resolves to +ROCm 10.3, which has no vLLM build and no row. One value flows the other way. A row's exact torchaudio pin, where it has one instead of a series, is also the hardware-verified build `rocm install sdk` pins diff --git a/engines/vllm/src/discover_builds.rs b/engines/vllm/src/discover_builds.rs index 637b8f0c96..5b3db2120b 100644 --- a/engines/vllm/src/discover_builds.rs +++ b/engines/vllm/src/discover_builds.rs @@ -50,6 +50,21 @@ impl DiscoverVersion { } } +/// Where a [`VllmRocmDiscoverBuild`] row gets its torch, torchvision and +/// torchaudio from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TorchStack { + /// Resolved from the row's `torch_index_url` by series, newest first. + Discover, + /// The versions the SDK install already put in the runtime, checked + /// against the row's series and ROCm line. For an index that serves + /// several ROCm lines at once, where the newest build of a series belongs + /// to the newest line: the nightly index lists about two dozen + /// `+rocm10.3.0aDATE` torch builds after the last `+rocm10.2.0a20261022` + /// one, and PEP 440 has no local-version wildcard to ask for the line. + FromInstalledSdk, +} + /// A ROCm SDK version whose vLLM/flash-attn/amd-aiter wheels aren't published /// under a fixed filename (AMD rotates the dev-tag suffix constantly), so the /// exact wheel must be discovered from the index at install time instead of @@ -90,6 +105,7 @@ pub(crate) struct VllmRocmDiscoverBuild { /// [`verified_torchaudio_release`](crate::wheel_builds::verified_torchaudio_release): set one only for a build confirmed to /// load against the row's torch on real hardware, never as a guess. pub(crate) torchaudio_version: DiscoverVersion, + pub(crate) torch_stack: TorchStack, } /// Every ROCm SDK `major.minor` line the engine installs by live wheel @@ -123,6 +139,7 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // establish, and this row has no coverage on the self-hosted lanes. // `rocm install sdk` reads this field too: see `torchaudio_version`. torchaudio_version: DiscoverVersion::Series("2.11"), + torch_stack: TorchStack::Discover, }, VllmRocmDiscoverBuild { rocm_sdk_version: "10.1.0", @@ -158,6 +175,7 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // confirmed-working builds instead. `rocm install sdk` pins this same // build for the 10.1 line: see `torchaudio_version`. torchaudio_version: DiscoverVersion::Exact("2.11.0.2"), + torch_stack: TorchStack::Discover, }, VllmRocmDiscoverBuild { rocm_sdk_version: "10.2.0", @@ -165,11 +183,16 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // Same staging host as 10.1: it publishes the `.rocm102` vLLM builds // (`0.32.1.dev0+g.d.rocm102`) next to the 10.1 ones. vllm_index_url: "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/vllm/", - // Torch stack from the same staging host as vLLM, not `whl-next`: - // `whl-next` publishes no `+rocm10.2` torch, while staging carries a - // full 10.2 stack (`torch 2.14.0`, `torchvision 0.29.2` and - // `torchaudio 2.11.1`, each `+rocm10.2.0rcN`). - torch_index_url: "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/", + // The torch stack is the SDK's own, from the nightly index it was + // installed from (see `TorchStack::FromInstalledSdk`). Not the staging + // host's `+rocm10.2.0rcN` torch: that build requires + // `amd-torch-device-gfx*==2.14.0+rocm10.2.0rcN` and + // `rocm==10.2.0rc2`, which staging answers 403 for, so a GPU payload + // for it cannot be installed, and it is a different release than the + // `10.2.0aDATE` SDK that `rocm install sdk` installs. The 10.2 vLLM + // wheel imports (`import vllm._C`) against the SDK's own torch + // `2.14.0+rocm10.2.0a20261022`. + torch_index_url: "https://nightly.repo.amd.com/rocm/whl-next/", vllm_version_prefix: "0.32", flash_attn_version_prefix: "2.8", amd_aiter_version_prefix: "0.1", @@ -178,10 +201,11 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // `_ZN3c104cuda19CUDAErrorLogCaptureC1Ev`; 2.14 imports cleanly. torch_version_prefix: "2.14", torchvision_version_prefix: "0.29", - // A series match, which resolves the newest `2.11` build, `2.11.1`. - // Not Exact, because an Exact pin would make `rocm install sdk` treat - // it as verified for a 10.2 SDK, which nobody has tested. + // A series match, not Exact: an Exact pin would make `rocm install + // sdk` treat it as verified for a 10.2 SDK, which nobody has run on a + // GPU. torchaudio_version: DiscoverVersion::Series("2.11"), + torch_stack: TorchStack::FromInstalledSdk, }, ]; @@ -313,11 +337,10 @@ mod tests { // 2.14, not 2.13: vLLM 0.32's `_C` needs a torch 2.14 symbol. assert_eq!(ten_two.torch_version_prefix, "2.14"); assert_eq!(ten_two.torchvision_version_prefix, "0.29"); - // The 10.2 torch stack is published on the staging host, unlike 10.1's. - assert_eq!( - ten_two.torch_index_url, - "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/" - ); + // The nightly index serves 10.2 and 10.3 torch side by side, so the + // 10.2 row reuses the SDK's own stack; the other rows discover theirs. + assert_eq!(ten_two.torch_stack, TorchStack::FromInstalledSdk); + assert_eq!(ten_one.torch_stack, TorchStack::Discover); // An unpublished line must not borrow another line's wheels. assert!(vllm_rocm_discover_build("10.3.0").is_none()); diff --git a/engines/vllm/src/install.rs b/engines/vllm/src/install.rs index ecd1ffc377..933bfab40c 100644 --- a/engines/vllm/src/install.rs +++ b/engines/vllm/src/install.rs @@ -13,7 +13,7 @@ use std::path::{Path, PathBuf}; use std::process::Command as ProcessCommand; use crate::discover_builds::{ - DiscoverVersion, VllmRocmDiscoverBuild, rocm_sdk_series_matches, rocm_series, + DiscoverVersion, TorchStack, VllmRocmDiscoverBuild, rocm_sdk_series_matches, rocm_series, vllm_rocm_discover_build, }; use crate::repair::{RepairAssessment, assess_runtime_repair}; @@ -567,6 +567,93 @@ fn discover_pinned_requirement_for( ) }) } +/// The row's torch, torchvision and torchaudio pins, in that order. +/// +/// A [`TorchStack::Discover`] row resolves them from its index. A +/// [`TorchStack::FromInstalledSdk`] row takes them from the runtime the SDK +/// install already resolved, because the index it realigns from serves +/// several ROCm lines at once and a series such as `torch==2.14.*` would pick +/// the newest line's build (see [`TorchStack`]). +fn torch_stack_pins( + uv: &Path, + paths: &AppPaths, + python: &Path, + build: &VllmRocmDiscoverBuild, +) -> Result> { + match build.torch_stack { + TorchStack::Discover => ["torch", "torchvision", "torchaudio"] + .into_iter() + .map(|pkg| { + let version = match pkg { + "torch" => DiscoverVersion::Series(build.torch_version_prefix), + "torchvision" => DiscoverVersion::Series(build.torchvision_version_prefix), + _ => build.torchaudio_version, + }; + discover_pinned_requirement(uv, paths, python, build.torch_index_url, pkg, version) + }) + .collect(), + TorchStack::FromInstalledSdk => { + let output = ProcessCommand::new(uv) + .args(uv_pip_freeze_args(python)) + .envs(uv_command_env(paths)) + .output() + .context("failed to launch `uv pip freeze` to read the SDK's torch stack")?; + if !output.status.success() { + bail!( + "`uv pip freeze` failed for {}: {}", + python.display(), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + installed_torch_stack_pins(&String::from_utf8_lossy(&output.stdout), build) + } + } +} + +/// [`torch_stack_pins`] for a [`TorchStack::FromInstalledSdk`] row, from the +/// `uv pip freeze` output of the SDK runtime. +/// +/// Each pin must belong to the row's torch, torchvision and torchaudio series +/// and be built for the row's ROCm line, so a runtime whose torch is not the +/// one the vLLM wheel was built against is refused instead of silently kept. +fn installed_torch_stack_pins(freeze: &str, build: &VllmRocmDiscoverBuild) -> Result> { + let wanted = [ + ("torch", build.torch_version_prefix), + ("torchvision", build.torchvision_version_prefix), + ( + "torchaudio", + match build.torchaudio_version { + DiscoverVersion::Series(series) | DiscoverVersion::Exact(series) => series, + }, + ), + ]; + wanted + .into_iter() + .map(|(pkg, series)| { + let prefix = format!("{pkg}=="); + let pin = freeze + .lines() + .map(str::trim) + .find(|line| line.starts_with(&prefix)) + .with_context(|| { + format!( + "the ROCm SDK runtime has no `{pkg}` installed; vLLM for ROCm {} reuses \ + the SDK's own torch stack, so reinstall the SDK with `rocm install sdk`", + build.rocm_sdk_version + ) + })?; + let release = &pin[prefix.len()..]; + if !release.starts_with(&format!("{series}.")) { + bail!( + "the ROCm SDK runtime has `{pin}`, but vLLM for ROCm {} needs `{pkg}` {series}.x", + build.rocm_sdk_version + ); + } + ensure_rocm_local_version_matches(pin, build.rocm_sdk_version)?; + Ok(pin.to_owned()) + }) + .collect() +} /// Runs the dependency-resolving `uv pip install --dry-run` for the discovered /// vLLM pin and returns uv's report, without installing anything. /// @@ -970,36 +1057,17 @@ fn install_vllm_rocm10_discover( build: &VllmRocmDiscoverBuild, ) -> Result { ensure_discover_python_tag(python, build)?; - let torch = discover_pinned_requirement( - uv, - paths, - python, - build.torch_index_url, - "torch", - DiscoverVersion::Series(build.torch_version_prefix), - )?; - // Discovered fresh from the same index and row as torch, rather than - // snapshotted from whatever the SDK install resolved earlier: that - // snapshot can pair with a *different* torch version than the one this - // row pins (the SDK's own install is unpinned and may have picked the - // newest torch/torchvision/torchaudio available, independently of each - // other), producing an ABI-incompatible triplet at realign time. - let torchvision = discover_pinned_requirement( - uv, - paths, - python, - build.torch_index_url, - "torchvision", - DiscoverVersion::Series(build.torchvision_version_prefix), - )?; - let torchaudio = discover_pinned_requirement( - uv, - paths, - python, - build.torch_index_url, - "torchaudio", - build.torchaudio_version, - )?; + // Discovered (or read from the SDK runtime) together, as one coherent set, + // rather than snapshotted from whatever the SDK install resolved earlier + // for a row that discovers: that snapshot can pair with a *different* + // torch version than the one this row pins, producing an ABI-incompatible + // triplet at realign time. + let mut torch_stack = torch_stack_pins(uv, paths, python, build)?.into_iter(); + let (torch, torchvision, torchaudio) = ( + torch_stack.next().context("torch pin")?, + torch_stack.next().context("torchvision pin")?, + torch_stack.next().context("torchaudio pin")?, + ); let vllm = discover_pinned_requirement( uv, paths, @@ -2153,6 +2221,39 @@ fi .is_err() ); } + /// A 10.2 runtime keeps the torch stack its SDK install resolved; a stack + /// from another series or another ROCm line is refused, not kept. + #[test] + fn installed_torch_stack_pins_reads_and_checks_the_sdk_stack() { + let ten_two = vllm_rocm_discover_build("10.2.0").expect("10.2 row"); + let freeze = "flash-attn==2.8.3\ntorch==2.14.0+rocm10.2.0a20261022\n\ + torchaudio==2.11.1+rocm10.2.0a20261022\n\ + torchvision==0.29.2+rocm10.2.0a20261022\n"; + assert_eq!( + installed_torch_stack_pins(freeze, ten_two).expect("coherent 10.2 stack"), + [ + "torch==2.14.0+rocm10.2.0a20261022", + "torchvision==0.29.2+rocm10.2.0a20261022", + "torchaudio==2.11.1+rocm10.2.0a20261022", + ] + ); + // The 10.1 torch an unpinned SDK install can leave behind is not 2.14's pair. + let old = freeze.replace("2.14.0+rocm10.2.0a20261022", "2.13.0+rocm10.2.0a20261022"); + let err = installed_torch_stack_pins(&old, ten_two).expect_err("torch 2.13"); + assert!( + format!("{err:#}").contains("needs `torch` 2.14.x"), + "{err:#}" + ); + // Another ROCm line is refused by the same guard as every other pin. + let other_line = freeze.replace( + "torch==2.14.0+rocm10.2.0a20261022", + "torch==2.14.0+rocm10.3.0a20261110", + ); + assert!(installed_torch_stack_pins(&other_line, ten_two).is_err()); + // A runtime with no torch says what to do. + let err = installed_torch_stack_pins("flash-attn==2.8.3\n", ten_two).expect_err("no torch"); + assert!(format!("{err:#}").contains("rocm install sdk"), "{err:#}"); + } /// Pins `install_vllm_rocm10_discover`'s own call sequencing, not just its /// arg-builder helpers in isolation: the remaining-deps install must use /// `--config-file` (never `--extra-index-url`, which cannot tolerate the From e755d0e2b55795bbd5255163865cd3fc7059cff8 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 17 Nov 2026 14:32:26 +0200 Subject: [PATCH 4/7] fix(vllm): fetch the 10.2 row's torch stack from the index the SDK came from The 10.2 row reuses the SDK's own torch, torchvision and torchaudio but fetched those exact builds again from the row's fixed nightly index. A release SDK's torch is not on that index, so the realign step would fail once 10.2 ships through the release channel. Read the index the runtime manifest records for the interpreter and use it for the torch install and both realign steps. A manifest that recorded none keeps the row's index, and an unreadable registry does the same but leaves a note. Rows that discover their stack are unchanged. Signed-off-by: juhovainio --- docs/vllm.md | 9 +- engines/vllm/src/discover_builds.rs | 6 +- engines/vllm/src/install.rs | 237 ++++++++++++++++++++++++++-- engines/vllm/src/runtime.rs | 59 +++++++ 4 files changed, 295 insertions(+), 16 deletions(-) diff --git a/docs/vllm.md b/docs/vllm.md index 4e47e070e8..8b99cc6442 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -123,12 +123,15 @@ The `10.2.0` row does not discover its torch stack: it keeps the torch, torchvision and torchaudio the SDK install already resolved (`torch 2.14.*`, torchvision `0.29.*`, torchaudio `2.11.*`, all `+rocm10.2.0aDATE`), checks them against the row's series and the SDK's ROCm line, and realigns to those exact -builds from the nightly index. A series match would not work there: that index -serves 10.2 and 10.3 builds side by side and the newest `torch 2.14.*` is a +builds from the index the SDK was installed from (the one its runtime manifest +records, so a nightly SDK uses the nightly index and a release SDK the release +one; the row's nightly index when a manifest recorded none). A series match +would not work there: the nightly index serves 10.2 and 10.3 builds side by side and the newest `torch 2.14.*` is a `+rocm10.3` build. The staging host's `+rocm10.2.0rcN` torch is not used: it requires `rocm==10.2.0rc2` and `amd-torch-device-gfx*==...rc2` payloads that staging does not serve. The row has been installed and imported (`import -vllm._C`) against a 10.2 SDK, but not yet served on a GPU, and the 10.2 vLLM +vllm._C`) against a 10.2 SDK. A GPU serve has been run on WSL2 only with the +runtime linked to its own `librocdxg.so`, never against a release SDK, and the 10.2 vLLM wheel is published for `linux_x86_64` only. The nightly channel resolves to ROCm 10.3, which has no vLLM build and no row. diff --git a/engines/vllm/src/discover_builds.rs b/engines/vllm/src/discover_builds.rs index 5b3db2120b..d6cdd77326 100644 --- a/engines/vllm/src/discover_builds.rs +++ b/engines/vllm/src/discover_builds.rs @@ -183,8 +183,10 @@ pub(crate) const VLLM_ROCM_DISCOVER_BUILD_TABLE: &[VllmRocmDiscoverBuild] = &[ // Same staging host as 10.1: it publishes the `.rocm102` vLLM builds // (`0.32.1.dev0+g.d.rocm102`) next to the 10.1 ones. vllm_index_url: "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/vllm/", - // The torch stack is the SDK's own, from the nightly index it was - // installed from (see `TorchStack::FromInstalledSdk`). Not the staging + // The torch stack is the SDK's own, fetched again from the index its + // runtime manifest records (see `TorchStack::FromInstalledSdk`); the + // nightly index below is only the fallback for a manifest that + // recorded none. Not the staging // host's `+rocm10.2.0rcN` torch: that build requires // `amd-torch-device-gfx*==2.14.0+rocm10.2.0rcN` and // `rocm==10.2.0rc2`, which staging answers 403 for, so a GPU payload diff --git a/engines/vllm/src/install.rs b/engines/vllm/src/install.rs index 933bfab40c..29039c9378 100644 --- a/engines/vllm/src/install.rs +++ b/engines/vllm/src/install.rs @@ -19,8 +19,8 @@ use crate::discover_builds::{ use crate::repair::{RepairAssessment, assess_runtime_repair}; use crate::runtime::{ VllmRuntime, assessed_python_for_repair, describe_skipped_managed_runtimes, - resolve_managed_runtime_python, resolve_vllm_runtime, runtime_is_managed, - vllm_runtime_warnings, + recorded_sdk_index_url, resolve_managed_runtime_python, resolve_vllm_runtime, + runtime_is_managed, vllm_runtime_warnings, }; use crate::state::runtime_lock_hash; use crate::wheel_builds::{ @@ -771,6 +771,19 @@ fn dry_run_resolved_pin(stdout: &str, pkg: &str) -> Option { (!version.is_empty()).then(|| format!("{pkg}=={version}")) }) } +/// The index the torch stack is fetched from for `build`. +/// +/// A [`TorchStack::FromInstalledSdk`] row reuses the SDK's own builds, so it asks for +/// them from the index the SDK came from (`recorded`, from the runtime manifest) rather +/// than the row's fixed one: the row serves nightly and release SDKs alike, and each +/// one's torch lives only on its own index. A manifest that recorded none, and every +/// row that discovers its stack, keeps the row's `torch_index_url`. +fn torch_index_url_for(build: &VllmRocmDiscoverBuild, recorded: Option<&str>) -> String { + match (build.torch_stack, recorded) { + (TorchStack::FromInstalledSdk, Some(url)) => url.to_owned(), + _ => build.torch_index_url.to_owned(), + } +} /// Builds the `uv pip install` argv for pinned torch alone, scoped to only /// `torch_index_url`. /// @@ -800,7 +813,7 @@ fn dry_run_resolved_pin(stdout: &str, pkg: &str) -> Option { fn vllm_rocm10_discover_torch_install_args( python: &Path, reinstall: bool, - build: &VllmRocmDiscoverBuild, + torch_index_url: &str, torch_pin: &str, ) -> Vec { let mut args = uv_pip_install_base(python); @@ -813,7 +826,7 @@ fn vllm_rocm10_discover_torch_install_args( args.push("--prerelease".to_owned()); args.push("allow".to_owned()); args.push("--index-url".to_owned()); - args.push(build.torch_index_url.to_owned()); + args.push(torch_index_url.to_owned()); args } /// Builds the `uv pip install` argv that realigns torch, torchvision and @@ -835,7 +848,7 @@ fn vllm_rocm10_discover_torch_install_args( /// exists to prevent. fn vllm_rocm10_discover_realign_install_args( python: &Path, - build: &VllmRocmDiscoverBuild, + torch_index_url: &str, pins: &[String], ) -> Vec { let mut args = uv_pip_install_base(python); @@ -850,7 +863,7 @@ fn vllm_rocm10_discover_realign_install_args( args.push("--prerelease".to_owned()); args.push("allow".to_owned()); args.push("--index-url".to_owned()); - args.push(build.torch_index_url.to_owned()); + args.push(torch_index_url.to_owned()); args } /// Builds the `uv pip install` argv for pinned vllm/flash-attn/amd-aiter, @@ -1122,7 +1135,17 @@ fn install_vllm_rocm10_discover( // Two separate `uv pip install` calls, each scoped to exactly one index: // see `vllm_rocm10_discover_torch_install_args` for why combining both // indexes into one call is unsafe. - let torch_args = vllm_rocm10_discover_torch_install_args(python, reinstall, build, &pins[0]); + let recorded_index = recorded_sdk_index_url(paths, python).unwrap_or_else(|err| { + notes.push(format!( + "could not read the runtime registry ({err:#}); the torch stack is fetched from {}, \ + which may not be the index the SDK was installed from", + build.torch_index_url + )); + None + }); + let torch_index_url = torch_index_url_for(build, recorded_index.as_deref()); + let torch_args = + vllm_rocm10_discover_torch_install_args(python, reinstall, &torch_index_url, &pins[0]); run_uv_pip_install(uv, paths, python, torch_args)?; let mut remaining_args = @@ -1149,7 +1172,8 @@ fn install_vllm_rocm10_discover( // force the whole stack back to the coherent trio discovered above. This // must run unconditionally, before the device-plugin discovery below: a // `uv pip freeze` failure there must not skip restoring the trio too. - let trio_realign_args = vllm_rocm10_discover_realign_install_args(python, build, &pins[..3]); + let trio_realign_args = + vllm_rocm10_discover_realign_install_args(python, &torch_index_url, &pins[..3]); run_uv_pip_install(uv, paths, python, trio_realign_args)?; // The full-dependency resolve can also pull in an AMD device-kernel @@ -1178,7 +1202,7 @@ fn install_vllm_rocm10_discover( } if !device_pins.is_empty() { let device_realign_args = - vllm_rocm10_discover_realign_install_args(python, build, &device_pins); + vllm_rocm10_discover_realign_install_args(python, &torch_index_url, &device_pins); run_uv_pip_install(uv, paths, python, device_realign_args)?; } @@ -1734,7 +1758,12 @@ mod tests { let build = vllm_rocm_discover_build("10.0.0").expect("10.0.0 has a discover row"); let torch_pin = "torch==2.12.0+rocm10.0.0"; - let args = vllm_rocm10_discover_torch_install_args(&python, false, build, torch_pin); + let args = vllm_rocm10_discover_torch_install_args( + &python, + false, + build.torch_index_url, + torch_pin, + ); assert!(!args.contains(&"--reinstall".to_owned())); assert!(!args.contains(&"--reinstall-package".to_owned())); assert!(args.contains(&torch_pin.to_owned())); @@ -1754,12 +1783,50 @@ mod tests { and hit a fatal 403 from a nonexistent sub-path" ); - let args = vllm_rocm10_discover_torch_install_args(&python, true, build, torch_pin); + let args = vllm_rocm10_discover_torch_install_args( + &python, + true, + build.torch_index_url, + torch_pin, + ); assert!(args.contains(&"--reinstall-package".to_owned())); assert!(args.contains(&"torch".to_owned())); assert!(!args.contains(&"--reinstall".to_owned())); } #[test] + fn an_installed_sdk_row_fetches_its_torch_stack_from_the_index_the_sdk_came_from() { + let python = PathBuf::from("/opt/venv/bin/python"); + let release = "https://stable.repo.amd.com/rocm/whl-next"; + let ten_two = vllm_rocm_discover_build("10.2.0").expect("10.2 has a discover row"); + assert_eq!(ten_two.torch_stack, TorchStack::FromInstalledSdk); + + // A release SDK's torch is not on the row's nightly index, so the row's own + // index must not be the one handed to uv. + let index = torch_index_url_for(ten_two, Some(release)); + assert_eq!(index, release); + assert_ne!(index, ten_two.torch_index_url); + let pins = vec!["torch==2.14.0+rocm10.2.0".to_owned()]; + let install = vllm_rocm10_discover_torch_install_args(&python, false, &index, &pins[0]); + let realign = vllm_rocm10_discover_realign_install_args(&python, &index, &pins); + for args in [&install, &realign] { + assert!(args.contains(&release.to_owned()), "{args:?}"); + assert!( + !args.contains(&ten_two.torch_index_url.to_owned()), + "{args:?}" + ); + } + + // A manifest that recorded no index keeps the row's. + assert_eq!(torch_index_url_for(ten_two, None), ten_two.torch_index_url); + // A row that discovers its stack never takes the recorded index. + let ten_one = vllm_rocm_discover_build("10.1.0").expect("10.1 has a discover row"); + assert_eq!(ten_one.torch_stack, TorchStack::Discover); + assert_eq!( + torch_index_url_for(ten_one, Some(release)), + ten_one.torch_index_url + ); + } + #[test] fn vllm_rocm10_discover_remaining_install_args_use_only_the_vllm_index() { let pins = vec![ "vllm==0.27.1.dev5+rocm10.0.0".to_owned(), @@ -2006,6 +2073,154 @@ exit 0 Ok((outcome, calls)) } + /// Runs `install_vllm_rocm10_discover` on the 10.2 row against a fake `uv` + /// whose `pip freeze` reports a release SDK's torch stack, with `prepare` + /// free to set up the runtime registry first. Returns the outcome and every + /// `uv` argv. + #[cfg(unix)] + fn run_ten_two_discover_with_fake_uv( + prepare: impl FnOnce(&AppPaths, &Path) -> Result<()>, + ) -> Result<(Result, Vec)> { + use rocm_core::fake_executable::write_fake_executable; + let root = tempfile::tempdir()?; + let paths = AppPaths { + config_dir: root.path().join("config"), + data_dir: root.path().join("data"), + cache_dir: root.path().join("cache"), + }; + let python = root.path().join("python"); + write_fake_executable(&python, "#!/bin/sh\necho cp314\n")?; + prepare(&paths, &python)?; + let log = root.path().join("uv.log"); + let uv = root.path().join("uv"); + write_fake_executable( + &uv, + format!( + r#"#!/bin/sh +echo "$@" >> "{log}" +last="" +for a in "$@"; do last="$a"; done +case "$*" in + *--dry-run*--config-file*) + echo "Resolved 3 packages" >&2 + ;; + *--dry-run*) + pkg=${{last%%==*}} + case "$last" in + *'*'*) echo " + ${{pkg}}==9.9.9" >&2 ;; + *) echo " + ${{last}}" >&2 ;; + esac + ;; + *"pip freeze"*) + echo "torch==2.14.0+rocm10.2.0" + echo "torchvision==0.29.2+rocm10.2.0" + echo "torchaudio==2.11.1+rocm10.2.0" + echo "amd-torch-device-gfx1151==2.14.0+rocm10.2.0" + ;; +esac +exit 0 +"#, + log = log.display() + ), + )?; + let build = vllm_rocm_discover_build("10.2.0").expect("10.2.0 has a discover row"); + let outcome = install_vllm_rocm10_discover(&uv, &paths, &python, true, build); + // A test may have made the registry unreadable; let the temp dir clean up. + let registry = paths.data_dir.join("runtimes").join("registry"); + if registry.is_dir() { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(®istry, std::fs::Permissions::from_mode(0o755))?; + } + let calls = std::fs::read_to_string(&log)? + .lines() + .map(str::to_owned) + .collect(); + Ok((outcome, calls)) + } + + #[cfg(unix)] + #[test] + fn a_release_sdk_on_the_ten_two_row_gets_its_torch_stack_from_its_own_index() -> Result<()> { + let release = "https://stable.example/rocm/whl-next"; + let (outcome, calls) = run_ten_two_discover_with_fake_uv(|paths, python| { + let registry = paths.data_dir.join("runtimes").join("registry"); + std::fs::create_dir_all(®istry)?; + std::fs::write( + registry.join("release.json"), + format!( + r#"{{"python_executable":"{}","index_url":"{release}"}}"#, + python.display() + ), + )?; + Ok(()) + })?; + outcome?; + let build = vllm_rocm_discover_build("10.2.0").expect("10.2.0 has a discover row"); + // The torch install, the trio realign and the device realign are the real + // installs that name a torch-stack package; each must ask the SDK's index. + let torch_stack_installs: Vec<&String> = calls + .iter() + .filter(|call| !call.contains("--dry-run") && call.contains("--no-deps")) + .filter(|call| call.contains("torch")) + .collect(); + assert_eq!( + torch_stack_installs.len(), + 3, + "expected torch install, trio realign and device realign: {calls:?}" + ); + for call in torch_stack_installs { + assert!(call.contains(&format!("--index-url {release}")), "{call}"); + assert!(!call.contains(build.torch_index_url), "{call}"); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn the_ten_two_row_keeps_its_own_index_without_a_recorded_one_and_says_when_it_could_not_look() + -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let build = vllm_rocm_discover_build("10.2.0").expect("10.2.0 has a discover row"); + + // A manifest that records no index: the row's index, no note. + let (outcome, calls) = run_ten_two_discover_with_fake_uv(|paths, python| { + let registry = paths.data_dir.join("runtimes").join("registry"); + std::fs::create_dir_all(®istry)?; + std::fs::write( + registry.join("old.json"), + format!(r#"{{"python_executable":"{}"}}"#, python.display()), + )?; + Ok(()) + })?; + assert!(outcome?.notes.is_empty()); + assert!( + calls + .iter() + .any(|call| call.contains(&format!("--index-url {}", build.torch_index_url))) + ); + + // A registry that cannot be read: the row's index, and the install says so. + let (outcome, calls) = run_ten_two_discover_with_fake_uv(|paths, _| { + let registry = paths.data_dir.join("runtimes").join("registry"); + std::fs::create_dir_all(®istry)?; + std::fs::set_permissions(®istry, std::fs::Permissions::from_mode(0o000))?; + Ok(()) + })?; + let notes = outcome?.notes; + assert!( + notes + .iter() + .any(|note| note.contains("could not read the runtime registry")), + "{notes:?}" + ); + assert!( + calls + .iter() + .any(|call| call.contains(&format!("--index-url {}", build.torch_index_url))) + ); + Ok(()) + } + #[cfg(unix)] #[test] fn discovery_installs_the_companions_the_discovered_vllm_depends_on() -> Result<()> { diff --git a/engines/vllm/src/runtime.rs b/engines/vllm/src/runtime.rs index 66971ad161..228ce53da0 100644 --- a/engines/vllm/src/runtime.rs +++ b/engines/vllm/src/runtime.rs @@ -78,6 +78,10 @@ pub(crate) struct TheRockRuntimeManifest { pub sdk_torch: Option, #[serde(default)] pub installed_at_unix_ms: Option, + /// The wheel index the SDK install resolved its packages from (nightly or + /// release); absent in manifests written before it was recorded. + #[serde(default)] + pub index_url: Option, } #[derive(Debug, Clone, Default, Deserialize)] @@ -356,6 +360,29 @@ fn sort_manifests_newest_install_first(manifests: &mut [(PathBuf, TheRockRuntime .then_with(|| left_path.cmp(right_path)) }); } +/// The wheel index the SDK in the runtime whose interpreter is `python` was installed +/// from, as its manifest in `paths`' runtime registry records it. `Ok(None)` means no +/// manifest for that interpreter recorded one; `Err` means the registry could not be read. +/// +/// A discover row that reuses the SDK's own torch stack has to fetch those exact builds +/// again from the index they came from: a nightly SDK's torch is not on the release +/// index and a release SDK's is not on the nightly one. +pub(crate) fn recorded_sdk_index_url(paths: &AppPaths, python: &Path) -> Result> { + let registry = paths.data_dir.join("runtimes").join("registry"); + let manifests = load_runtime_manifests_in(®istry, None)?; + Ok(sdk_index_url_for_python(&manifests, python)) +} +/// The `index_url` of the manifest whose interpreter is `python`, if it recorded a +/// non-empty one. +fn sdk_index_url_for_python(manifests: &[TheRockRuntimeManifest], python: &Path) -> Option { + manifests + .iter() + .find(|manifest| manifest.python_executable.as_deref() == Some(python)) + .and_then(|manifest| manifest.index_url.as_deref()) + .map(str::trim) + .filter(|url| !url.is_empty()) + .map(str::to_owned) +} /// The torch build the SDK installed into this runtime, as its manifest records it. /// /// Read from the manifest, never from the environment. By the time this runs the @@ -845,6 +872,38 @@ fn dedupe_paths(entries: Vec) -> Vec { mod tests { use super::*; + #[test] + fn the_sdk_index_comes_from_the_manifest_of_that_interpreter() { + let manifest = |python: &str, index: Option<&str>| TheRockRuntimeManifest { + python_executable: Some(PathBuf::from(python)), + index_url: index.map(str::to_owned), + ..Default::default() + }; + let manifests = vec![ + manifest( + "/rt/nightly/bin/python", + Some("https://nightly.example/whl"), + ), + manifest( + "/rt/release/bin/python", + Some(" https://stable.example/whl "), + ), + manifest("/rt/old/bin/python", None), + manifest("/rt/blank/bin/python", Some(" ")), + ]; + let lookup = |python: &str| sdk_index_url_for_python(&manifests, Path::new(python)); + assert_eq!( + lookup("/rt/nightly/bin/python").as_deref(), + Some("https://nightly.example/whl") + ); + assert_eq!( + lookup("/rt/release/bin/python").as_deref(), + Some("https://stable.example/whl") + ); + assert_eq!(lookup("/rt/old/bin/python"), None); + assert_eq!(lookup("/rt/blank/bin/python"), None); + assert_eq!(lookup("/rt/unknown/bin/python"), None); + } #[test] fn managed_env_reflects_managed_runtime_manifest_source() { let mut runtime = VllmRuntime { From 07058295db05f4c3d2b25fa8907ed0768c3df805 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 17 Nov 2026 14:49:50 +0200 Subject: [PATCH 5/7] fix(vllm): honour exact torch pins and bare +rocmNNN, and correct the discover-rows doc Review fixes for the 10.2 discover row: - installed_torch_stack_pins read an exact torchaudio pin as a series prefix of a version that still carried its +local part, so an Exact pin rejected the release it names and accepted longer ones. An exact pin now means that release, a series keeps its prefix rule. - ensure_rocm_local_version_matches sent a local version starting with rocm to the dotted parser only, so a bare +rocm102 was rejected. The dotted spelling is now read from its first rocm segment wherever it sits, and the compact one is the fallback. - Assert the 10.2 row's torch_index_url, which stays the fallback index. - docs/vllm.md claimed all three rows share the stable index and resolve their torch pins fresh; that holds for 10.0 and 10.1 only. Signed-off-by: juhovainio --- docs/vllm.md | 17 ++--- engines/vllm/src/discover_builds.rs | 7 +++ engines/vllm/src/install.rs | 98 ++++++++++++++++++++++++----- 3 files changed, 97 insertions(+), 25 deletions(-) diff --git a/docs/vllm.md b/docs/vllm.md index 8b99cc6442..3de0e460af 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -110,14 +110,15 @@ vLLM install then refuses that runtime; re-running the install with `--python-version 3.14` moves it back. This route is selected by `major.minor`, so `10.0.0`, `10.1.0` and `10.2.0` -discover through *different* rows: they pin different vLLM minors and read vLLM from -different index URLs, because AMD stages ROCm 10.1's vLLM builds on a separate -host from 10.0's production index. Both rows take their torch stack from the -same `whl-next` index, which is also where `rocm install sdk` resolves either -SDK's own `+rocmX.Y` torch from. vLLM's discovery is independent of that: it -resolves its own torch, torchvision, and torchaudio pins fresh from the row's -own static version prefixes, instead of reusing what the SDK install resolved. -The SDK install might have resolved a different torch version than the row pins. +discover through *different* rows, each pinning its own vLLM minor. `10.0.0` reads +vLLM from the production index; `10.1.0` and `10.2.0` read it from a separate +staging host, where AMD stages their vLLM builds. The `10.0.0` and `10.1.0` rows +take their torch stack from the stable `whl-next` index, which is also where +`rocm install sdk` resolves those SDKs' own `+rocmX.Y` torch from. For those two +rows vLLM's discovery is independent of the SDK install: it resolves its own +torch, torchvision, and torchaudio pins fresh from the row's own static version +prefixes, instead of reusing what the SDK install resolved. The SDK install might +have resolved a different torch version than the row pins. The `10.2.0` row does not discover its torch stack: it keeps the torch, torchvision and torchaudio the SDK install already resolved (`torch 2.14.*`, diff --git a/engines/vllm/src/discover_builds.rs b/engines/vllm/src/discover_builds.rs index d6cdd77326..f1629323b5 100644 --- a/engines/vllm/src/discover_builds.rs +++ b/engines/vllm/src/discover_builds.rs @@ -69,6 +69,7 @@ pub(crate) enum TorchStack { /// under a fixed filename (AMD rotates the dev-tag suffix constantly), so the /// exact wheel must be discovered from the index at install time instead of /// pinned in `wheel_builds::VLLM_ROCM_BUILD_TABLE`. +#[derive(Clone, Copy)] pub(crate) struct VllmRocmDiscoverBuild { pub(crate) rocm_sdk_version: &'static str, /// PEP 425 interpreter tag this row's wheels are published for. AMD builds @@ -335,6 +336,12 @@ mod tests { ten_two.vllm_index_url, "https://rocm.frameworks-prereleases.amd.com/whl-multi-arch-staging/vllm/" ); + // Still read by the torch install and the realign for the 10.2 row, as the + // fallback when the runtime manifest records no index of its own. + assert_eq!( + ten_two.torch_index_url, + "https://nightly.repo.amd.com/rocm/whl-next/" + ); assert_eq!(ten_two.vllm_version_prefix, "0.32"); // 2.14, not 2.13: vLLM 0.32's `_C` needs a torch 2.14 symbol. assert_eq!(ten_two.torch_version_prefix, "2.14"); diff --git a/engines/vllm/src/install.rs b/engines/vllm/src/install.rs index 29039c9378..6ceb6dfcce 100644 --- a/engines/vllm/src/install.rs +++ b/engines/vllm/src/install.rs @@ -610,6 +610,15 @@ fn torch_stack_pins( } } +/// Whether an installed `release` (a `uv pip freeze` version, with any `+local` part) +/// satisfies `wanted`: a series takes any release inside it, an exact pin takes that +/// release and no longer one, with the same meaning as PEP 440 `==release`. +fn stack_release_matches(release: &str, wanted: DiscoverVersion) -> bool { + match wanted { + DiscoverVersion::Series(series) => release.starts_with(&format!("{series}.")), + DiscoverVersion::Exact(exact) => split_local_version(release).0 == exact, + } +} /// [`torch_stack_pins`] for a [`TorchStack::FromInstalledSdk`] row, from the /// `uv pip freeze` output of the SDK runtime. /// @@ -618,18 +627,16 @@ fn torch_stack_pins( /// one the vLLM wheel was built against is refused instead of silently kept. fn installed_torch_stack_pins(freeze: &str, build: &VllmRocmDiscoverBuild) -> Result> { let wanted = [ - ("torch", build.torch_version_prefix), - ("torchvision", build.torchvision_version_prefix), + ("torch", DiscoverVersion::Series(build.torch_version_prefix)), ( - "torchaudio", - match build.torchaudio_version { - DiscoverVersion::Series(series) | DiscoverVersion::Exact(series) => series, - }, + "torchvision", + DiscoverVersion::Series(build.torchvision_version_prefix), ), + ("torchaudio", build.torchaudio_version), ]; wanted .into_iter() - .map(|(pkg, series)| { + .map(|(pkg, wanted_release)| { let prefix = format!("{pkg}=="); let pin = freeze .lines() @@ -643,10 +650,14 @@ fn installed_torch_stack_pins(freeze: &str, build: &VllmRocmDiscoverBuild) -> Re ) })?; let release = &pin[prefix.len()..]; - if !release.starts_with(&format!("{series}.")) { + if !stack_release_matches(release, wanted_release) { bail!( - "the ROCm SDK runtime has `{pin}`, but vLLM for ROCm {} needs `{pkg}` {series}.x", - build.rocm_sdk_version + "the ROCm SDK runtime has `{pin}`, but vLLM for ROCm {} needs `{pkg}` {}", + build.rocm_sdk_version, + match wanted_release { + DiscoverVersion::Series(series) => format!("{series}.x"), + DiscoverVersion::Exact(release) => release.to_owned(), + } ); } ensure_rocm_local_version_matches(pin, build.rocm_sdk_version)?; @@ -960,13 +971,7 @@ fn ensure_rocm_local_version_matches(pin: &str, rocm_sdk_version: &str) -> Resul let Some(local) = split_local_version(version).1 else { return Ok(()); }; - let matches = if let Some(local_rocm) = local.strip_prefix("rocm") { - rocm_sdk_series_matches(local_rocm, rocm_sdk_version) - } else if let Some(compact) = compact_rocm_local_segment(local) { - // `.rocm102` names 10.2 with no dot between major and minor. - rocm_series(rocm_sdk_version) - .is_some_and(|(major, minor)| compact == format!("{major}{minor}")) - } else { + let Some(matches) = local_rocm_matches(local, rocm_sdk_version) else { return Ok(()); }; if matches { @@ -977,6 +982,29 @@ fn ensure_rocm_local_version_matches(pin: &str, rocm_sdk_version: &str) -> Resul different ROCm release; the index for this ROCm line is serving another SDK's wheels" ) } +/// Whether the ROCm line a local version names is `rocm_sdk_version`'s, or `None` when +/// it names none. A dotted `rocm10.2.0` is read from its first `rocm` segment to the end +/// of the local version, wherever it sits; a compact `rocm102` is the digits of that +/// segment alone, so `rocm102` as the whole local version and `d20261110.rocm102` after +/// other segments are both read. +fn local_rocm_matches(local: &str, rocm_sdk_version: &str) -> Option { + let segments: Vec<&str> = local.split('.').collect(); + let at = segments + .iter() + .position(|segment| segment.starts_with("rocm"))?; + let dotted = segments[at..].join("."); + if let Some(recorded) = dotted.strip_prefix("rocm") + && rocm_series(recorded).is_some() + { + return Some(rocm_sdk_series_matches(recorded, rocm_sdk_version)); + } + let compact = compact_rocm_local_segment(local)?; + // `.rocm102` names 10.2 with no dot between major and minor. + Some( + rocm_series(rocm_sdk_version) + .is_some_and(|(major, minor)| compact == format!("{major}{minor}")), + ) +} /// The digits of a `rocm` segment in a dotted local version, e.g. /// `102` for `g5df63a720.d20261110.rocm102`: the spelling the 10.2 vLLM wheels /// use, with no dot between the ROCm major and minor. @@ -2430,6 +2458,14 @@ fi let err = ensure_rocm_local_version_matches(ten_two, "10.1.0") .expect_err("a 10.2 wheel must not install onto a 10.1 SDK"); assert!(format!("{err:#}").contains("rocm102"), "{err:#}"); + // A tagged release carries the compact spelling as the whole local version. + let bare = "vllm==0.32.1+rocm102"; + assert!(ensure_rocm_local_version_matches(bare, "10.2.0").is_ok()); + assert!(ensure_rocm_local_version_matches(bare, "10.1.0").is_err()); + // And the dotted spelling is read as a series wherever its segment sits. + let late = "vllm==0.32.1+g5df63a720.rocm10.2.0"; + assert!(ensure_rocm_local_version_matches(late, "10.2.0").is_ok()); + assert!(ensure_rocm_local_version_matches(late, "10.3.0").is_err()); // And a 10.1 wheel must not slip onto a 10.2 SDK through the dotted spelling. assert!( ensure_rocm_local_version_matches("vllm==0.29.1.dev0+rocm10.1.0.gabc", "10.2.0") @@ -2469,6 +2505,34 @@ fi let err = installed_torch_stack_pins("flash-attn==2.8.3\n", ten_two).expect_err("no torch"); assert!(format!("{err:#}").contains("rocm install sdk"), "{err:#}"); } + /// An `Exact` torchaudio pin on a row that reuses the SDK's stack means that + /// release and no other (PEP 440 `==release`): the `+local` part is not + /// compared, and a longer release is not a match. + #[test] + fn installed_torch_stack_pins_treats_an_exact_pin_as_that_release() { + let ten_two = vllm_rocm_discover_build("10.2.0").expect("10.2 row"); + let exact = |release: &'static str| VllmRocmDiscoverBuild { + torchaudio_version: DiscoverVersion::Exact(release), + ..*ten_two + }; + let freeze = |torchaudio: &str| { + format!( + "torch==2.14.0+rocm10.2.0a20261022\ntorchvision==0.29.2+rocm10.2.0a20261022\n\ + torchaudio=={torchaudio}+rocm10.2.0a20261022\n" + ) + }; + let accepted = installed_torch_stack_pins(&freeze("2.11.0.2"), &exact("2.11.0.2")) + .expect("the exact release with its local version"); + assert_eq!(accepted[2], "torchaudio==2.11.0.2+rocm10.2.0a20261022"); + for installed in ["2.11.0.2.9", "2.11.0.3", "2.11.1"] { + assert!( + installed_torch_stack_pins(&freeze(installed), &exact("2.11.0.2")).is_err(), + "{installed} is not the release 2.11.0.2" + ); + } + // A shorter pin does not accept the longer rebuild it is a prefix of. + assert!(installed_torch_stack_pins(&freeze("2.11.0.2"), &exact("2.11.0")).is_err()); + } /// Pins `install_vllm_rocm10_discover`'s own call sequencing, not just its /// arg-builder helpers in isolation: the remaining-deps install must use /// `--config-file` (never `--extra-index-url`, which cannot tolerate the From 5527cbde7bd463ce5f92dfb10c2d2c862b214a61 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 17 Nov 2026 15:29:26 +0200 Subject: [PATCH 6/7] fix(vllm): reject a rocm segment no spelling reads, and say when the torch index falls back Follow-up review fixes: - A local version with a rocm segment that is neither the dotted nor the compact spelling (+rocm, +rocmabc) passed the guard since the last change; it is a mismatch again, as it was before. - Only the row that reuses the SDK's torch stack looks up the recorded index, so an unreadable registry no longer adds a note to 10.0 and 10.1 installs, whose index is fixed by design. - That row also says so when the manifest records no index, and every failure on the torch install and the realigns carries the fallback notes, not just the remaining-deps install. - The unreadable-registry test skips itself where mode 000 does not block the reader (running as root). - docs/release-trust.md: an overridden pip base is recorded in the manifest and reused by later vLLM installs for that runtime. Signed-off-by: juhovainio --- docs/release-trust.md | 7 ++ docs/vllm.md | 3 +- engines/vllm/src/install.rs | 238 +++++++++++++++++++++++++++--------- 3 files changed, 190 insertions(+), 58 deletions(-) diff --git a/docs/release-trust.md b/docs/release-trust.md index 8f9f4678ad..d12575da33 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -291,6 +291,13 @@ Set the gate and the specific base variable together, and only in tests or deliberate manual QA against a fixture server. The `therock-next` E2E scenarios use exactly this pairing to exercise next-layout dispatch hermetically. +A wheel install records the pip index it resolved from, including an overridden +base, in the runtime's manifest as `index_url`. `rocm engines install vllm` reads +that value back for a ROCm 10.2 runtime and fetches the SDK's own torch stack from +it, without checking its scheme or host, so an override used for one install keeps +being used for that runtime's later vLLM installs, after the override is unset. +Remove and reinstall the runtime, or edit its manifest, to stop that. + ## ComfyUI Source Archive Override rocm-cli hardcodes the URL it downloads the ComfyUI source archive from. It diff --git a/docs/vllm.md b/docs/vllm.md index 3de0e460af..d1b828dd89 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -126,7 +126,8 @@ torchvision `0.29.*`, torchaudio `2.11.*`, all `+rocm10.2.0aDATE`), checks them against the row's series and the SDK's ROCm line, and realigns to those exact builds from the index the SDK was installed from (the one its runtime manifest records, so a nightly SDK uses the nightly index and a release SDK the release -one; the row's nightly index when a manifest recorded none). A series match +one; the row's nightly index when a manifest recorded none or the registry could +not be read, and the install output says which). A series match would not work there: the nightly index serves 10.2 and 10.3 builds side by side and the newest `torch 2.14.*` is a `+rocm10.3` build. The staging host's `+rocm10.2.0rcN` torch is not used: it requires `rocm==10.2.0rc2` and `amd-torch-device-gfx*==...rc2` payloads that diff --git a/engines/vllm/src/install.rs b/engines/vllm/src/install.rs index 6ceb6dfcce..60ebf4723e 100644 --- a/engines/vllm/src/install.rs +++ b/engines/vllm/src/install.rs @@ -782,6 +782,19 @@ fn dry_run_resolved_pin(stdout: &str, pkg: &str) -> Option { (!version.is_empty()).then(|| format!("{pkg}=={version}")) }) } +/// `err` with the fallbacks this install used appended, so a failure on a fallback +/// names it. The cause comes first, so plain `{}` display leads with the uv failure and +/// not with a note about how the pins or the index were chosen. +fn with_fallback_notes(err: anyhow::Error, notes: &[String]) -> anyhow::Error { + if notes.is_empty() { + err + } else { + anyhow!( + "{err:#}\n\nFallbacks used during this install: {}", + notes.join(" ") + ) + } +} /// The index the torch stack is fetched from for `build`. /// /// A [`TorchStack::FromInstalledSdk`] row reuses the SDK's own builds, so it asks for @@ -983,7 +996,7 @@ fn ensure_rocm_local_version_matches(pin: &str, rocm_sdk_version: &str) -> Resul ) } /// Whether the ROCm line a local version names is `rocm_sdk_version`'s, or `None` when -/// it names none. A dotted `rocm10.2.0` is read from its first `rocm` segment to the end +/// it has no `rocm` segment at all. A dotted `rocm10.2.0` is read from its first `rocm` segment to the end /// of the local version, wherever it sits; a compact `rocm102` is the digits of that /// segment alone, so `rocm102` as the whole local version and `d20261110.rocm102` after /// other segments are both read. @@ -998,12 +1011,13 @@ fn local_rocm_matches(local: &str, rocm_sdk_version: &str) -> Option { { return Some(rocm_sdk_series_matches(recorded, rocm_sdk_version)); } - let compact = compact_rocm_local_segment(local)?; - // `.rocm102` names 10.2 with no dot between major and minor. - Some( + // `.rocm102` names 10.2 with no dot between major and minor. A `rocm` segment + // that is neither spelling (`+rocm`, `+rocmabc`) names no line this SDK can be + // matched to, so it is a mismatch, not a pass. + Some(compact_rocm_local_segment(local).is_some_and(|compact| { rocm_series(rocm_sdk_version) - .is_some_and(|(major, minor)| compact == format!("{major}{minor}")), - ) + .is_some_and(|(major, minor)| compact == format!("{major}{minor}")) + })) } /// The digits of a `rocm` segment in a dotted local version, e.g. /// `102` for `g5df63a720.d20261110.rocm102`: the spelling the 10.2 vLLM wheels @@ -1163,37 +1177,38 @@ fn install_vllm_rocm10_discover( // Two separate `uv pip install` calls, each scoped to exactly one index: // see `vllm_rocm10_discover_torch_install_args` for why combining both // indexes into one call is unsafe. - let recorded_index = recorded_sdk_index_url(paths, python).unwrap_or_else(|err| { - notes.push(format!( - "could not read the runtime registry ({err:#}); the torch stack is fetched from {}, \ - which may not be the index the SDK was installed from", - build.torch_index_url - )); - None - }); - let torch_index_url = torch_index_url_for(build, recorded_index.as_deref()); + let torch_index_url = if build.torch_stack == TorchStack::FromInstalledSdk { + let recorded_index = recorded_sdk_index_url(paths, python); + match &recorded_index { + Ok(Some(_)) => {} + Ok(None) => notes.push(format!( + "the runtime manifest records no wheel index for this SDK; the torch stack is \ + fetched from {}, which may not be the index the SDK was installed from", + build.torch_index_url + )), + Err(err) => notes.push(format!( + "could not read the runtime registry ({err:#}); the torch stack is fetched \ + from {}, which may not be the index the SDK was installed from", + build.torch_index_url + )), + } + torch_index_url_for(build, recorded_index.ok().flatten().as_deref()) + } else { + build.torch_index_url.to_owned() + }; let torch_args = vllm_rocm10_discover_torch_install_args(python, reinstall, &torch_index_url, &pins[0]); - run_uv_pip_install(uv, paths, python, torch_args)?; + run_uv_pip_install(uv, paths, python, torch_args) + .map_err(|err| with_fallback_notes(err, ¬es))?; let mut remaining_args = vllm_rocm10_discover_remaining_install_args(python, reinstall, &pins[3..]); remaining_args.push("--config-file".to_owned()); remaining_args.push(config_file.display().to_string()); - if let Err(err) = run_uv_pip_install(uv, paths, python, remaining_args) { - // A note kept only for the success path would be lost exactly when the - // fallback pins are what failed. - return Err(if notes.is_empty() { - err - } else { - // The cause first, so plain `{}` display leads with the uv failure - // and not with a note about how the pins were chosen. - anyhow!( - "{err:#}\n\nThe fallback pins were used because: {}", - notes.join(" ") - ) - }); - } + // A note kept only for the success path would be lost exactly when the + // fallback it describes is what failed. + run_uv_pip_install(uv, paths, python, remaining_args) + .map_err(|err| with_fallback_notes(err, ¬es))?; // The full-dependency resolve above can replace torch (and, pulling it in // transitively, torchvision/torchaudio) with unconstrained PyPI builds; @@ -1202,7 +1217,8 @@ fn install_vllm_rocm10_discover( // `uv pip freeze` failure there must not skip restoring the trio too. let trio_realign_args = vllm_rocm10_discover_realign_install_args(python, &torch_index_url, &pins[..3]); - run_uv_pip_install(uv, paths, python, trio_realign_args)?; + run_uv_pip_install(uv, paths, python, trio_realign_args) + .map_err(|err| with_fallback_notes(err, ¬es))?; // The full-dependency resolve can also pull in an AMD device-kernel // plugin for any of the three (`amd-torch-device-gfx*`, @@ -1231,7 +1247,8 @@ fn install_vllm_rocm10_discover( if !device_pins.is_empty() { let device_realign_args = vllm_rocm10_discover_realign_install_args(python, &torch_index_url, &device_pins); - run_uv_pip_install(uv, paths, python, device_realign_args)?; + run_uv_pip_install(uv, paths, python, device_realign_args) + .map_err(|err| with_fallback_notes(err, ¬es))?; } let mut pins = pins; @@ -2108,6 +2125,16 @@ exit 0 #[cfg(unix)] fn run_ten_two_discover_with_fake_uv( prepare: impl FnOnce(&AppPaths, &Path) -> Result<()>, + ) -> Result<(Result, Vec)> { + run_discover_row_with_fake_uv("10.2.0", false, prepare) + } + /// [`run_ten_two_discover_with_fake_uv`] for any row, optionally with every + /// real `--no-deps` install (the torch install and the realigns) failing. + #[cfg(unix)] + fn run_discover_row_with_fake_uv( + row: &str, + fail_torch_stack_installs: bool, + prepare: impl FnOnce(&AppPaths, &Path) -> Result<()>, ) -> Result<(Result, Vec)> { use rocm_core::fake_executable::write_fake_executable; let root = tempfile::tempdir()?; @@ -2139,6 +2166,9 @@ case "$*" in *) echo " + ${{last}}" >&2 ;; esac ;; + *--no-deps*) + {fail} + ;; *"pip freeze"*) echo "torch==2.14.0+rocm10.2.0" echo "torchvision==0.29.2+rocm10.2.0" @@ -2148,10 +2178,15 @@ case "$*" in esac exit 0 "#, - log = log.display() + log = log.display(), + fail = if fail_torch_stack_installs { + "echo 'error: boom' >&2; exit 1" + } else { + "true" + } ), )?; - let build = vllm_rocm_discover_build("10.2.0").expect("10.2.0 has a discover row"); + let build = vllm_rocm_discover_build(row).expect("the row has a discover build"); let outcome = install_vllm_rocm10_discover(&uv, &paths, &python, true, build); // A test may have made the registry unreadable; let the temp dir clean up. let registry = paths.data_dir.join("runtimes").join("registry"); @@ -2203,48 +2238,126 @@ exit 0 Ok(()) } + /// Plants a runtime manifest for the interpreter, with `extra` JSON fields. #[cfg(unix)] - #[test] - fn the_ten_two_row_keeps_its_own_index_without_a_recorded_one_and_says_when_it_could_not_look() - -> Result<()> { + fn plant_manifest(paths: &AppPaths, python: &Path, extra: &str) -> Result<()> { + let registry = paths.data_dir.join("runtimes").join("registry"); + std::fs::create_dir_all(®istry)?; + std::fs::write( + registry.join("runtime.json"), + format!(r#"{{"python_executable":"{}"{extra}}}"#, python.display()), + )?; + Ok(()) + } + /// Makes the registry unreadable, or reports that it cannot be: the owner of a + /// mode-000 directory still reads it as root, so a root run skips what needs it. + #[cfg(unix)] + fn make_registry_unreadable(paths: &AppPaths) -> Result { use std::os::unix::fs::PermissionsExt; - let build = vllm_rocm_discover_build("10.2.0").expect("10.2.0 has a discover row"); + let registry = paths.data_dir.join("runtimes").join("registry"); + std::fs::create_dir_all(®istry)?; + std::fs::set_permissions(®istry, std::fs::Permissions::from_mode(0o000))?; + Ok(std::fs::read_dir(®istry).is_err()) + } - // A manifest that records no index: the row's index, no note. - let (outcome, calls) = run_ten_two_discover_with_fake_uv(|paths, python| { - let registry = paths.data_dir.join("runtimes").join("registry"); - std::fs::create_dir_all(®istry)?; - std::fs::write( - registry.join("old.json"), - format!(r#"{{"python_executable":"{}"}}"#, python.display()), - )?; - Ok(()) - })?; - assert!(outcome?.notes.is_empty()); - assert!( + #[cfg(unix)] + #[test] + fn the_ten_two_row_says_when_it_falls_back_to_its_own_index() -> Result<()> { + let build = vllm_rocm_discover_build("10.2.0").expect("10.2.0 has a discover row"); + let on_row_index = |calls: &[String]| { calls .iter() .any(|call| call.contains(&format!("--index-url {}", build.torch_index_url))) + }; + + // A manifest that records no index (an older one, or an adopted runtime): + // the row's index, and the install says so. + let (outcome, calls) = + run_ten_two_discover_with_fake_uv(|paths, python| plant_manifest(paths, python, ""))?; + let notes = outcome?.notes; + assert!( + notes + .iter() + .any(|note| note.contains("records no wheel index")), + "{notes:?}" ); + assert!(on_row_index(&calls)); - // A registry that cannot be read: the row's index, and the install says so. + // A registry that cannot be read: the same index, and the install says so. + let mut skipped = false; let (outcome, calls) = run_ten_two_discover_with_fake_uv(|paths, _| { - let registry = paths.data_dir.join("runtimes").join("registry"); - std::fs::create_dir_all(®istry)?; - std::fs::set_permissions(®istry, std::fs::Permissions::from_mode(0o000))?; + skipped = !make_registry_unreadable(paths)?; + Ok(()) + })?; + if !skipped { + let notes = outcome?.notes; + assert!( + notes + .iter() + .any(|note| note.contains("could not read the runtime registry")), + "{notes:?}" + ); + assert!(on_row_index(&calls)); + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_row_that_discovers_its_stack_never_looks_at_the_registry_or_says_anything_about_it() + -> Result<()> { + // 10.1's index is fixed by design, so an unreadable registry or a manifest + // with no index is not something its install has a reason to mention. + let mut skipped = false; + let (outcome, _) = run_discover_row_with_fake_uv("10.1.0", false, |paths, _| { + skipped = !make_registry_unreadable(paths)?; Ok(()) })?; let notes = outcome?.notes; + let _ = skipped; // the assertion holds whether or not the registry was unreadable assert!( notes .iter() - .any(|note| note.contains("could not read the runtime registry")), + .all(|note| !note.contains("registry") && !note.contains("wheel index")), "{notes:?}" ); + let (outcome, _) = run_discover_row_with_fake_uv("10.1.0", false, |paths, python| { + plant_manifest(paths, python, "") + })?; assert!( - calls + outcome? + .notes .iter() - .any(|call| call.contains(&format!("--index-url {}", build.torch_index_url))) + .all(|note| !note.contains("wheel index")) + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_failed_torch_stack_install_on_the_fallback_index_names_the_fallback() -> Result<()> { + let (outcome, _) = run_discover_row_with_fake_uv("10.2.0", true, |paths, python| { + plant_manifest(paths, python, "") + })?; + let err = format!("{:#}", outcome.expect_err("the torch install fails")); + assert!(err.contains("boom"), "{err}"); + assert!( + err.contains("records no wheel index") && err.contains("Fallbacks used"), + "the uv failure must carry the note about the index it ran against: {err}" + ); + + // With the SDK's own index recorded there is nothing to add to the failure. + let (outcome, _) = run_discover_row_with_fake_uv("10.2.0", true, |paths, python| { + plant_manifest( + paths, + python, + r#","index_url":"https://stable.example/whl""#, + ) + })?; + let err = format!("{:#}", outcome.expect_err("the torch install fails")); + assert!( + err.contains("boom") && !err.contains("Fallbacks used"), + "{err}" ); Ok(()) } @@ -2462,6 +2575,17 @@ fi let bare = "vllm==0.32.1+rocm102"; assert!(ensure_rocm_local_version_matches(bare, "10.2.0").is_ok()); assert!(ensure_rocm_local_version_matches(bare, "10.1.0").is_err()); + // A `rocm` segment that is neither spelling names no line, so it is no match. + for unreadable in [ + "vllm==0.32.1+rocm", + "vllm==0.32.1+rocmabc", + "vllm==0.32.1+g5d.rocm.x", + ] { + assert!( + ensure_rocm_local_version_matches(unreadable, "10.2.0").is_err(), + "{unreadable}" + ); + } // And the dotted spelling is read as a series wherever its segment sits. let late = "vllm==0.32.1+g5df63a720.rocm10.2.0"; assert!(ensure_rocm_local_version_matches(late, "10.2.0").is_ok()); From d0d5e0ea494ede28ba4b1b194df3df583f7d1a8a Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 17 Nov 2026 16:40:51 +0200 Subject: [PATCH 7/7] refactor(vllm): attach the fallback notes to every failed discover step, and test them Review nits on the 10.2 discover row: - The notes about a fallback (companion pins, the torch index) were attached to individual failures one call at a time, so errors before the torch install and between the realigns still lost them. The sequence now records them as it goes and the entry point attaches them once, to whatever fails. - Test each real install step failing on the fallback index, not just the first. - The unreadable-registry case plants a directory where a manifest file belongs, so it fails for root too and no longer needs a skip. - Say 'no runtime manifest for this interpreter records a wheel index', which also covers an interpreter with no manifest at all. Signed-off-by: juhovainio --- docs/vllm.md | 4 +- engines/vllm/src/install.rs | 154 +++++++++++++++++++----------------- 2 files changed, 84 insertions(+), 74 deletions(-) diff --git a/docs/vllm.md b/docs/vllm.md index d1b828dd89..ac1c73a639 100644 --- a/docs/vllm.md +++ b/docs/vllm.md @@ -126,8 +126,8 @@ torchvision `0.29.*`, torchaudio `2.11.*`, all `+rocm10.2.0aDATE`), checks them against the row's series and the SDK's ROCm line, and realigns to those exact builds from the index the SDK was installed from (the one its runtime manifest records, so a nightly SDK uses the nightly index and a release SDK the release -one; the row's nightly index when a manifest recorded none or the registry could -not be read, and the install output says which). A series match +one; the row's nightly index when no manifest for the SDK's interpreter recorded +one or the registry could not be read, and the install output says which). A series match would not work there: the nightly index serves 10.2 and 10.3 builds side by side and the newest `torch 2.14.*` is a `+rocm10.3` build. The staging host's `+rocm10.2.0rcN` torch is not used: it requires `rocm==10.2.0rc2` and `amd-torch-device-gfx*==...rc2` payloads that diff --git a/engines/vllm/src/install.rs b/engines/vllm/src/install.rs index 60ebf4723e..377a5f681e 100644 --- a/engines/vllm/src/install.rs +++ b/engines/vllm/src/install.rs @@ -1111,6 +1111,25 @@ fn install_vllm_rocm10_discover( reinstall: bool, build: &VllmRocmDiscoverBuild, ) -> Result { + let mut notes: Vec = Vec::new(); + // Every failure after a fallback was taken names it, wherever it happens: a note + // kept only for the success path would be lost exactly when the fallback it + // describes is what failed. + match install_vllm_rocm10_discover_collecting(uv, paths, python, reinstall, build, &mut notes) { + Ok(pins) => Ok(VllmInstallOutcome { pins, notes }), + Err(err) => Err(with_fallback_notes(err, ¬es)), + } +} +/// [`install_vllm_rocm10_discover`]'s sequence, recording each fallback it takes in +/// `notes` as it goes and returning the pins installed. +fn install_vllm_rocm10_discover_collecting( + uv: &Path, + paths: &AppPaths, + python: &Path, + reinstall: bool, + build: &VllmRocmDiscoverBuild, + notes: &mut Vec, +) -> Result> { ensure_discover_python_tag(python, build)?; // Discovered (or read from the SDK runtime) together, as one coherent set, // rather than snapshotted from whatever the SDK install resolved earlier @@ -1136,7 +1155,6 @@ fn install_vllm_rocm10_discover( // not fail the install: the real install below is the authority, and it // reports its own error. The reason is kept, because the series match is // exactly the choice this probe exists to improve on. - let mut notes: Vec = Vec::new(); let vllm_resolution = match resolve_vllm_dependencies(uv, paths, python, &vllm, &config_file) { Ok(resolution) => Some(resolution), Err(err) => { @@ -1182,7 +1200,7 @@ fn install_vllm_rocm10_discover( match &recorded_index { Ok(Some(_)) => {} Ok(None) => notes.push(format!( - "the runtime manifest records no wheel index for this SDK; the torch stack is \ + "no runtime manifest for this interpreter records a wheel index; the torch stack is \ fetched from {}, which may not be the index the SDK was installed from", build.torch_index_url )), @@ -1198,17 +1216,13 @@ fn install_vllm_rocm10_discover( }; let torch_args = vllm_rocm10_discover_torch_install_args(python, reinstall, &torch_index_url, &pins[0]); - run_uv_pip_install(uv, paths, python, torch_args) - .map_err(|err| with_fallback_notes(err, ¬es))?; + run_uv_pip_install(uv, paths, python, torch_args)?; let mut remaining_args = vllm_rocm10_discover_remaining_install_args(python, reinstall, &pins[3..]); remaining_args.push("--config-file".to_owned()); remaining_args.push(config_file.display().to_string()); - // A note kept only for the success path would be lost exactly when the - // fallback it describes is what failed. - run_uv_pip_install(uv, paths, python, remaining_args) - .map_err(|err| with_fallback_notes(err, ¬es))?; + run_uv_pip_install(uv, paths, python, remaining_args)?; // The full-dependency resolve above can replace torch (and, pulling it in // transitively, torchvision/torchaudio) with unconstrained PyPI builds; @@ -1217,8 +1231,7 @@ fn install_vllm_rocm10_discover( // `uv pip freeze` failure there must not skip restoring the trio too. let trio_realign_args = vllm_rocm10_discover_realign_install_args(python, &torch_index_url, &pins[..3]); - run_uv_pip_install(uv, paths, python, trio_realign_args) - .map_err(|err| with_fallback_notes(err, ¬es))?; + run_uv_pip_install(uv, paths, python, trio_realign_args)?; // The full-dependency resolve can also pull in an AMD device-kernel // plugin for any of the three (`amd-torch-device-gfx*`, @@ -1247,13 +1260,12 @@ fn install_vllm_rocm10_discover( if !device_pins.is_empty() { let device_realign_args = vllm_rocm10_discover_realign_install_args(python, &torch_index_url, &device_pins); - run_uv_pip_install(uv, paths, python, device_realign_args) - .map_err(|err| with_fallback_notes(err, ¬es))?; + run_uv_pip_install(uv, paths, python, device_realign_args)?; } let mut pins = pins; pins.extend(device_pins); - Ok(VllmInstallOutcome { pins, notes }) + Ok(pins) } #[cfg(test)] mod tests { @@ -2126,14 +2138,15 @@ exit 0 fn run_ten_two_discover_with_fake_uv( prepare: impl FnOnce(&AppPaths, &Path) -> Result<()>, ) -> Result<(Result, Vec)> { - run_discover_row_with_fake_uv("10.2.0", false, prepare) + run_discover_row_with_fake_uv("10.2.0", None, prepare) } - /// [`run_ten_two_discover_with_fake_uv`] for any row, optionally with every - /// real `--no-deps` install (the torch install and the realigns) failing. + /// [`run_ten_two_discover_with_fake_uv`] for any row, optionally failing the + /// `fail_install`th real `uv pip install` (1 torch, 2 remaining deps, 3 trio + /// realign, 4 device realign; the dry-runs and `pip freeze` are not counted). #[cfg(unix)] fn run_discover_row_with_fake_uv( row: &str, - fail_torch_stack_installs: bool, + fail_install: Option, prepare: impl FnOnce(&AppPaths, &Path) -> Result<()>, ) -> Result<(Result, Vec)> { use rocm_core::fake_executable::write_fake_executable; @@ -2155,6 +2168,15 @@ exit 0 echo "$@" >> "{log}" last="" for a in "$@"; do last="$a"; done +case "$*" in + *--dry-run*|*"pip freeze"*) ;; + *) + n=$(cat "{count}" 2>/dev/null || echo 0) + n=$((n + 1)) + echo "$n" > "{count}" + if [ "$n" = "{fail_at}" ]; then echo 'error: boom' >&2; exit 1; fi + ;; +esac case "$*" in *--dry-run*--config-file*) echo "Resolved 3 packages" >&2 @@ -2166,9 +2188,6 @@ case "$*" in *) echo " + ${{last}}" >&2 ;; esac ;; - *--no-deps*) - {fail} - ;; *"pip freeze"*) echo "torch==2.14.0+rocm10.2.0" echo "torchvision==0.29.2+rocm10.2.0" @@ -2179,21 +2198,12 @@ esac exit 0 "#, log = log.display(), - fail = if fail_torch_stack_installs { - "echo 'error: boom' >&2; exit 1" - } else { - "true" - } + count = root.path().join("uv.count").display(), + fail_at = fail_install.unwrap_or(0) ), )?; let build = vllm_rocm_discover_build(row).expect("the row has a discover build"); let outcome = install_vllm_rocm10_discover(&uv, &paths, &python, true, build); - // A test may have made the registry unreadable; let the temp dir clean up. - let registry = paths.data_dir.join("runtimes").join("registry"); - if registry.is_dir() { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(®istry, std::fs::Permissions::from_mode(0o755))?; - } let calls = std::fs::read_to_string(&log)? .lines() .map(str::to_owned) @@ -2249,15 +2259,17 @@ exit 0 )?; Ok(()) } - /// Makes the registry unreadable, or reports that it cannot be: the owner of a - /// mode-000 directory still reads it as root, so a root run skips what needs it. + /// Makes the registry unreadable: a directory where a manifest file belongs + /// cannot be read as a file by anyone, root included, unlike a mode-000 registry. #[cfg(unix)] - fn make_registry_unreadable(paths: &AppPaths) -> Result { - use std::os::unix::fs::PermissionsExt; - let registry = paths.data_dir.join("runtimes").join("registry"); - std::fs::create_dir_all(®istry)?; - std::fs::set_permissions(®istry, std::fs::Permissions::from_mode(0o000))?; - Ok(std::fs::read_dir(®istry).is_err()) + fn make_registry_unreadable(paths: &AppPaths) -> Result<()> { + let broken = paths + .data_dir + .join("runtimes") + .join("registry") + .join("broken.json"); + std::fs::create_dir_all(broken)?; + Ok(()) } #[cfg(unix)] @@ -2278,27 +2290,22 @@ exit 0 assert!( notes .iter() - .any(|note| note.contains("records no wheel index")), + .any(|note| note.contains("records a wheel index")), "{notes:?}" ); assert!(on_row_index(&calls)); // A registry that cannot be read: the same index, and the install says so. - let mut skipped = false; - let (outcome, calls) = run_ten_two_discover_with_fake_uv(|paths, _| { - skipped = !make_registry_unreadable(paths)?; - Ok(()) - })?; - if !skipped { - let notes = outcome?.notes; - assert!( - notes - .iter() - .any(|note| note.contains("could not read the runtime registry")), - "{notes:?}" - ); - assert!(on_row_index(&calls)); - } + let (outcome, calls) = + run_ten_two_discover_with_fake_uv(|paths, _| make_registry_unreadable(paths))?; + let notes = outcome?.notes; + assert!( + notes + .iter() + .any(|note| note.contains("could not read the runtime registry")), + "{notes:?}" + ); + assert!(on_row_index(&calls)); Ok(()) } @@ -2308,20 +2315,17 @@ exit 0 -> Result<()> { // 10.1's index is fixed by design, so an unreadable registry or a manifest // with no index is not something its install has a reason to mention. - let mut skipped = false; - let (outcome, _) = run_discover_row_with_fake_uv("10.1.0", false, |paths, _| { - skipped = !make_registry_unreadable(paths)?; - Ok(()) + let (outcome, _) = run_discover_row_with_fake_uv("10.1.0", None, |paths, _| { + make_registry_unreadable(paths) })?; let notes = outcome?.notes; - let _ = skipped; // the assertion holds whether or not the registry was unreadable assert!( notes .iter() .all(|note| !note.contains("registry") && !note.contains("wheel index")), "{notes:?}" ); - let (outcome, _) = run_discover_row_with_fake_uv("10.1.0", false, |paths, python| { + let (outcome, _) = run_discover_row_with_fake_uv("10.1.0", None, |paths, python| { plant_manifest(paths, python, "") })?; assert!( @@ -2335,19 +2339,25 @@ exit 0 #[cfg(unix)] #[test] - fn a_failed_torch_stack_install_on_the_fallback_index_names_the_fallback() -> Result<()> { - let (outcome, _) = run_discover_row_with_fake_uv("10.2.0", true, |paths, python| { - plant_manifest(paths, python, "") - })?; - let err = format!("{:#}", outcome.expect_err("the torch install fails")); - assert!(err.contains("boom"), "{err}"); - assert!( - err.contains("records no wheel index") && err.contains("Fallbacks used"), - "the uv failure must carry the note about the index it ran against: {err}" - ); + fn a_failed_install_step_on_the_fallback_index_names_the_fallback() -> Result<()> { + // Each real install in turn: the torch install, the remaining deps, the trio + // realign and the device realign all run after the index fell back. + for step in 1..=4 { + let (outcome, _) = + run_discover_row_with_fake_uv("10.2.0", Some(step), |paths, python| { + plant_manifest(paths, python, "") + })?; + let err = format!("{:#}", outcome.expect_err("an install step fails")); + assert!(err.contains("boom"), "step {step}: {err}"); + assert!( + err.contains("records a wheel index") && err.contains("Fallbacks used"), + "step {step}: the uv failure must carry the note about the index it ran \ + against: {err}" + ); + } // With the SDK's own index recorded there is nothing to add to the failure. - let (outcome, _) = run_discover_row_with_fake_uv("10.2.0", true, |paths, python| { + let (outcome, _) = run_discover_row_with_fake_uv("10.2.0", Some(1), |paths, python| { plant_manifest( paths, python,