From 59205762ca85b6441063ba082b18500539f1c3ed Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 3 Nov 2026 15:08:29 +0200 Subject: [PATCH 01/13] fix(lemonade): pin Lemonade 2026.44.1 so a ROCm 10.1 SDK gets a ROCm backend Lemonade 11.5.1 pins TheRock 7.13.0 and llama.cpp b9752, and downloads TheRock from a location that does not serve ROCm 10.x, so on a ROCm 10.1 SDK its llamacpp:rocm backend could not be paired with the SDK in use. Lemonade 2026.44.1 pins TheRock 10.1.0 and llama.cpp b11580, whose release ships ROCm 10.1 (and 10.0 and 7.14) archives for Linux and Windows, and installs TheRock from AMD's whl-next wheel index or core tarball location. The embeddable layout, binary names and the lemond/lemonade flags rocm-cli uses are unchanged; the new archive digests are recorded. Not 2026.45.3, the newest stable release: its Linux lemond needs glibc 2.43 and OpenSSL 3.2.0 and exits with a loader error on Ubuntu 24.04, which has glibc 2.39 and OpenSSL 3.0.13. 2026.44.1's needs glibc 2.38 and OpenSSL 3.0.0, the same as every release back to 11.5.1. Two adaptations the new release line needs: - `lemonade backends` lists recipes rocm-cli did not know (llamacpp-hrx, ds4, thenoise) and adds Tier and Format columns. The parser matched recipe rows against a fixed list, so an unknown recipe's rows, and every indented row after them, were read as llamacpp backends. It now reads the recipe from the row layout, and reads Lemonade's update_required and update_available states as installable and installed instead of dropping the row. - The second alignment attempt took lemonade-sdk/llama.cpp's newest release, but a release can ship without some or all of its ROCm archives: b12013 through b12086 carried only qualification reports, and others skip one ROCm version or one platform. It now lists the 30 newest releases and picks the most recently published one that ships the exact archive for the active ROCm version on this platform. The runtime-16 and runtime-17 scenarios assumed the active SDK never equals the packaged pin, which is no longer true on a ROCm 10.1 lane: runtime-16 now checks that the installed pin equals the active SDK by whichever route the host takes, and runtime-17 compares the pin with the one inside the verified archive. Signed-off-by: Roman Inflianskas --- MANIFEST.md | 13 +- .../e2e-report/src/consolidated/markdown.rs | 4 +- crates/rocm-deps/src/lib.rs | 4 +- docs/engine-plugins.md | 30 +- docs/testing.md | 21 + engines/lemonade/src/backend_alignment.rs | 494 ++++++++++++++---- engines/lemonade/src/direct_llama.rs | 6 +- engines/lemonade/src/install.rs | 90 ++-- engines/lemonade/src/llamacpp_backend.rs | 164 +++++- runtime-deps.toml | 7 +- .../features/runtime_setup.feature | 44 +- tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 215 ++++++-- 12 files changed, 847 insertions(+), 245 deletions(-) diff --git a/MANIFEST.md b/MANIFEST.md index 9e9103156c..eaeaff3314 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -706,11 +706,16 @@ Lemonade embeddable archive from the official Lemonade SDK GitHub releases at to the rocm-cli managed data directory and is reused for subsequent `rocm engines start lemonade` invocations. This is a self-contained binary distribution; no Python packages are installed for Lemonade by rocm-cli. -On Linux/WSL, `rocm engines install lemonade` may also query -`https://api.github.com/repos/lemonade-sdk/llama.cpp/releases/latest` (an -unauthenticated call) while aligning Lemonade's `llamacpp:rocm` backend to -the active ROCm SDK version; see +On Linux/WSL and Windows, `rocm engines install lemonade` may also list +`https://api.github.com/repos/lemonade-sdk/llama.cpp/releases` (the 30 newest +releases, one unauthenticated call) while aligning Lemonade's `llamacpp:rocm` +backend to the active ROCm SDK version; see [Lemonade backend alignment on engine install](docs/engine-plugins.md#lemonade-backend-alignment-on-engine-install). +Lemonade itself then downloads the llama.cpp backend it installs from the +`lemonade-sdk/llama.cpp` GitHub releases, and the ROCm runtime that backend +runs on from AMD's ROCm package repository (as Python wheels into a virtual +environment of Lemonade's own, or as a TheRock tarball); those downloads are +Lemonade's, not rocm-cli's. ### Engine-Specific Python Dependencies diff --git a/crates/e2e-report/src/consolidated/markdown.rs b/crates/e2e-report/src/consolidated/markdown.rs index 60fb5a4362..f1f0c2f400 100644 --- a/crates/e2e-report/src/consolidated/markdown.rs +++ b/crates/e2e-report/src/consolidated/markdown.rs @@ -772,7 +772,7 @@ mod tests { let platform = r#"{ "platform_slug": "mi300x", "capability": {"effective_serve_engine": "vllm"}, - "versions": {"os":"Ubuntu 24.04.3 LTS","rocm":"7.13.0","vllm":"0.23.0+rocm723","lemonade":"11.5.1"}, + "versions": {"os":"Ubuntu 24.04.3 LTS","rocm":"7.13.0","vllm":"0.23.0+rocm723","lemonade":"2026.44.1"}, "expectations": [ {"id":"serve-x","effective_engine":"vllm","expected":"pass"} ] @@ -784,7 +784,7 @@ mod tests { "Ubuntu 24.04.3 LTS", "ROCm 7.13.0", "vLLM 0.23.0+rocm723", - "lemonade 11.5.1", + "lemonade 2026.44.1", ] { assert!(md.contains(token), "matrix cell missing {token:?}:\n{md}"); } diff --git a/crates/rocm-deps/src/lib.rs b/crates/rocm-deps/src/lib.rs index ee95efbf0d..281e36272e 100644 --- a/crates/rocm-deps/src/lib.rs +++ b/crates/rocm-deps/src/lib.rs @@ -17,14 +17,14 @@ include!(concat!(env!("OUT_DIR"), "/pins.rs")); /// GitHub repository publishing the Lemonade embeddable release archives. pub const LEMONADE_GITHUB_REPO: &str = "lemonade-sdk/lemonade"; -/// Strip a leading `v` from a release tag (`v11.5.1` -> `11.5.1`). +/// Strip a leading `v` from a release tag (`v2026.44.1` -> `2026.44.1`). #[must_use] pub fn strip_v(tag: &str) -> &str { tag.strip_prefix('v').unwrap_or(tag) } /// Name of the Lemonade embeddable archive for a host token and extension, -/// e.g. `lemonade-embeddable-11.5.1-ubuntu-x64.tar.gz`. +/// e.g. `lemonade-embeddable-2026.44.1-ubuntu-x64.tar.gz`. #[must_use] pub fn lemonade_archive_name(version: &str, os_arch: &str, extension: &str) -> String { format!( diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index 3cd9610703..58e313eb1e 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -54,14 +54,21 @@ The rewrite tries two builds in order. The first keeps Lemonade's own pinned llama.cpp release and just repoints it at the active ROCm version — a deliberate, reproducible pin. If that attempt fails to resolve against the installed backend's shared libraries — for example because that release never -shipped an asset for the active ROCm version — the second queries GitHub's -`releases/latest` for `lemonade-sdk/llama.cpp` (an unauthenticated -`api.github.com` call, subject to GitHub's public rate limit) and installs -whatever build is newest at that moment. That second build is a moving -target, not a pin: which llama.cpp commit actually gets installed depends on -when the install ran, and the version installed is not recorded anywhere -`rocm examine`/`rocm version` report. An attempt that cannot be verified -against the installed backend's shared libraries reverts to whichever version +shipped an asset for the active ROCm version — the second lists the 30 newest +`lemonade-sdk/llama.cpp` releases on GitHub (an unauthenticated +`api.github.com` call, subject to GitHub's public rate limit) and installs the +most recently published one that ships the llama.cpp archive for the active +ROCm version's `major.minor` on this platform +(`llama--bin-ubuntu-rocm--x64.tar.gz` on Linux, +`llama--bin-win-rocm--x64.zip` on Windows). The newest +release alone is not used, because a release can be published without its +ROCm archives. The second attempt is skipped when none of the 30 ships that +archive, when the listing cannot be fetched, and when the build it finds is the +one Lemonade already pins (the first attempt tried that build). That second +build is a moving target, not a pin: which llama.cpp commit actually gets +installed depends on when the install ran, and the version installed is not +recorded anywhere `rocm examine`/`rocm version` report. An attempt that cannot +be verified against the installed backend's shared libraries reverts to whichever version was pinned before the attempt — the packaged pin on a fresh install or after `--reinstall`, but whatever alignment last wrote otherwise — rather than being kept. @@ -132,8 +139,11 @@ The versions of the third-party runtimes rocm-cli downloads are pinned in `runtime-deps.toml` at the repository root — one `[runtime.]` table per runtime. That file is the only place a runtime version is written down: archive names, download URLs, and the dashboard's offline fallback are all -derived from it, so a bump is a one-line edit and the tree cannot end up -holding two different versions of the same runtime. +derived from it, so the tree cannot end up holding two different versions of +the same runtime. A Lemonade bump also records the new release's archive +digests (`EMBEDDABLE_LINUX_SHA256` and `EMBEDDABLE_WINDOWS_SHA256` in +[install.rs](../engines/lemonade/src/install.rs)): the downloaded archive is +checked against them, and a digest cannot be derived from the version. `rocm engines list` shows the exact plugin directories for the current host. The same output is available in the TUI with `/engine`. diff --git a/docs/testing.md b/docs/testing.md index baa36d0792..908068beeb 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2165,6 +2165,27 @@ lemonade` against the shared tree) reaches the restore only on a runner whose shared tree still holds a stale pin; on a healthy runner the install succeeds and the restore never runs, so a green lane does not show that it works. +Lemonade backend selection and alignment focused tests (reading the +`llamacpp` rows out of `lemonade backends` when other recipes have indented +rows of their own, and the second alignment attempt choosing the newest +`lemonade-sdk/llama.cpp` release that ships the archive for the active ROCm +version rather than the newest release, and reading Lemonade's +`update_required`/`update_available` backend states): + +```bash +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::parse +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::another_recipes_backend +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::update_states +cargo test -p rocm-engine-lemonade backend_alignment::tests::tier2 +cargo test -p rocm-engine-lemonade backend_alignment::tests::align_falls_through_to_tier2 +cargo test -p rocm-engine-lemonade backend_alignment::tests::llamacpp_asset_name +``` + +They run on fixtures: the `lemonade backends` table Lemonade 2026.44.1 prints, +and a release listing shaped like the real one. The live paths (a real +`lemond`, the GitHub listing, a real backend install) run only on the gated +Lemonade GPU lanes. + Engine recipe adapter contract focused tests: ```bash diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index 3899d32f9c..67586f4149 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -95,7 +95,7 @@ pub(crate) fn prepare_llamacpp_backend_for_active_rocm( disabled, try_llamacpp_backend_alignment, install_best_llamacpp_backend, - latest_llamacpp_rocm_stable_tag, + newest_llamacpp_rocm_stable_tag_for, ) }, ) @@ -341,7 +341,7 @@ fn install_pinned_backend_or_restore_packaged_pins( /// The Tier 1 / Tier 2 / revert state machine, isolated from the config and active-SDK /// lookups above so it can be exercised in tests against a temp `backend_versions.json` -/// with the install/align/latest-tag steps injected, instead of spawning a real +/// with the install/align/tag-lookup steps injected, instead of spawning a real /// `lemond` and reaching GitHub. /// /// `disabled` is the resolved value of [`lemonade_backend_alignment_disabled`], passed @@ -365,7 +365,7 @@ fn align_llamacpp_backend_to_version( disabled: bool, mut align: impl FnMut(&mut LemonadeInstallManifest, &str, bool, &str) -> bool, mut fallback_install: impl FnMut(&mut LemonadeInstallManifest, bool) -> Result<()>, - mut latest_tag: impl FnMut() -> Result, + mut newest_tag_with_asset: impl FnMut(&str) -> Result, ) -> Result> { if disabled { fallback_install(manifest, false)?; @@ -393,31 +393,41 @@ fn align_llamacpp_backend_to_version( // matching ROCm-version asset — Lemonade only started publishing per-ROCm- // version builds partway through its release history, and the embeddable's // pin (frozen at whatever build was current when this Lemonade version was - // cut) does not track that. Point at the current newest build instead, still - // paired with the active ROCm version, forcing a real reinstall attempt - // regardless of whatever Tier 1 left on disk. + // cut) does not track that. Point at the newest build that actually ships an + // archive for the active ROCm version instead, still paired with that version, + // forcing a real reinstall attempt regardless of whatever Tier 1 left on disk. let pinned_tag = read_backend_versions_llamacpp_tag(backend_versions_path); - let latest_tag_applied = match latest_tag() { - Ok(latest_tag) if pinned_tag.as_deref() != Some(latest_tag.as_str()) => { - match write_backend_versions_llamacpp_tag(backend_versions_path, &latest_tag) { + let found_tag_applied = match newest_tag_with_asset(target_version) { + Ok(tag) if pinned_tag.as_deref() != Some(tag.as_str()) => { + match write_backend_versions_llamacpp_tag(backend_versions_path, &tag) { Ok(()) => true, Err(error) => { eprintln!( - "Warning: could not pin Lemonade's backend to llama.cpp {latest_tag}: \ + "Warning: could not pin Lemonade's backend to llama.cpp {tag}: \ {error:#}" ); false } } } - // The pinned tag already is the latest — Tier 1 already tried it. + // The pinned tag already is that build — Tier 1 already tried it. Ok(_) => false, Err(error) => { - eprintln!("Warning: could not determine Lemonade's latest llama.cpp build: {error:#}"); + eprintln!( + "Warning: could not look up a llama.cpp build that ships an archive for ROCm \ + {target_version}: {error:#}" + ); false } }; - if latest_tag_applied && align(manifest, target_version, true, "the latest llama.cpp build") { + if found_tag_applied + && align( + manifest, + target_version, + true, + "the newest llama.cpp build for this ROCm version", + ) + { return Ok(Some(target_version.to_owned())); } @@ -440,7 +450,7 @@ fn align_llamacpp_backend_to_version( ({pinned_version}): {error:#}" ); } - if latest_tag_applied { + if found_tag_applied { // Best-effort, matching the therock.version restore above: a failure here // must not skip the fallback reinstall below. Restore the original tag when // there was one; otherwise Tier 2 pinned a key that did not exist before, so @@ -545,36 +555,117 @@ const fn llamacpp_backend_alignment_succeeded( /// `resources/backend_versions.json`'s `therock.version` alone doesn't get a /// ROCm-version-matched build if the pinned tag predates that ROCm version's /// support: Lemonade only started publishing per-ROCm-version assets partway -/// through this repo's release history, and the embeddable's own pin can be -/// (and, at the time of writing, is) older than that. +/// through this repo's release history, and an embeddable's own pin can be older +/// than that (Lemonade 11.5.1's b9752 was). const LLAMACPP_ROCM_STABLE_REPO: &str = "lemonade-sdk/llama.cpp"; -/// The newest published release tag for [`LLAMACPP_ROCM_STABLE_REPO`] (e.g. -/// `b10952`), queried directly rather than assumed — the whole reason Tier 2 -/// exists is that the embeddable's own pinned tag cannot be trusted to have a -/// matching-ROCm-version asset. -fn latest_llamacpp_rocm_stable_tag() -> Result { - let url = format!("https://api.github.com/repos/{LLAMACPP_ROCM_STABLE_REPO}/releases/latest"); - let timeout = Duration::from_secs(15); +/// How many of [`LLAMACPP_ROCM_STABLE_REPO`]'s newest releases Tier 2 searches +/// for one that ships the archive it needs. The repository publishes a build +/// about twice a day, so this covers roughly the last two weeks; one listing page +/// of this size is about 2 MB. +const LLAMACPP_RELEASES_SEARCHED: usize = 30; + +/// The `llamacpp:rocm-stable` archive Lemonade downloads for build `tag` paired +/// with ROCm `rocm_version` — the name Lemonade's own installer builds, from the +/// ROCm version's `major.minor` (`10.1.0` -> `rocm-10.1`). +fn llamacpp_rocm_stable_asset_name(tag: &str, rocm_version: &str, windows: bool) -> String { + let major_minor = rocm_version + .splitn(3, '.') + .take(2) + .collect::>() + .join("."); + if windows { + format!("llama-{tag}-bin-win-rocm-{major_minor}-x64.zip") + } else { + format!("llama-{tag}-bin-ubuntu-rocm-{major_minor}-x64.tar.gz") + } +} + +/// From a GitHub release listing (`GET /repos/{repo}/releases`), the tag of the +/// most recently published release that actually ships the `llamacpp:rocm-stable` +/// archive for `rocm_version` on this platform. +/// +/// The newest release is not good enough on its own: a release can be published +/// before, or without, its ROCm archives (`b12013` through `b12086` carry only +/// qualification reports), and a build can skip one ROCm version or one platform. +/// Drafts, prereleases, and assets not reported as `uploaded` are skipped. +fn newest_llamacpp_tag_with_asset( + releases: &Value, + rocm_version: &str, + windows: bool, +) -> Option { + releases + .as_array()? + .iter() + .filter(|release| { + !release + .get("draft") + .and_then(Value::as_bool) + .unwrap_or(false) + && !release + .get("prerelease") + .and_then(Value::as_bool) + .unwrap_or(false) + }) + .filter_map(|release| { + let tag = release.get("tag_name")?.as_str()?; + let wanted = llamacpp_rocm_stable_asset_name(tag, rocm_version, windows); + let ships_asset = release.get("assets")?.as_array()?.iter().any(|asset| { + asset.get("name").and_then(Value::as_str) == Some(wanted.as_str()) + && asset.get("state").and_then(Value::as_str) == Some("uploaded") + }); + let published_at = release.get("published_at")?.as_str()?; + ships_asset.then_some((published_at, tag)) + }) + .max_by(|left, right| left.0.cmp(right.0)) + .map(|(_, tag)| tag.to_owned()) +} + +/// The newest [`LLAMACPP_ROCM_STABLE_REPO`] build that ships an archive for +/// `rocm_version` on this platform (e.g. `b12003`), queried directly rather than +/// assumed — the whole reason Tier 2 exists is that the embeddable's own pinned +/// tag cannot be trusted to have a matching-ROCm-version asset. +fn newest_llamacpp_rocm_stable_tag_for(rocm_version: &str) -> Result { + newest_llamacpp_rocm_stable_tag_via(rocm_version, fetch_github_json) +} + +/// GET `url` from the GitHub API and parse the body as JSON. +fn fetch_github_json(url: &str) -> Result { // `timeout_connect` takes precedence over `timeout` and defaults to 30s, so without // it a host that blackholes rather than refuses would stall well past the intended - // ceiling. + // ceiling. The overall timeout also covers reading the listing. let agent = ureq::AgentBuilder::new() - .timeout_connect(timeout) - .timeout(timeout) + .timeout_connect(Duration::from_secs(15)) + .timeout(Duration::from_secs(30)) .build(); let response = agent - .get(&url) + .get(url) .set("User-Agent", "rocm-cli") .call() .with_context(|| format!("failed to query {url}"))?; - let body: Value = response + response .into_json() - .with_context(|| format!("failed to parse response from {url}"))?; - body.get("tag_name") - .and_then(Value::as_str) - .map(str::to_owned) - .with_context(|| format!("{url} response is missing 'tag_name'")) + .with_context(|| format!("failed to parse response from {url}")) +} + +/// [`newest_llamacpp_rocm_stable_tag_for`] with the listing fetch injected, so +/// a test reaches the host-platform choice without the network. +fn newest_llamacpp_rocm_stable_tag_via( + rocm_version: &str, + fetch: impl FnOnce(&str) -> Result, +) -> Result { + let url = format!( + "https://api.github.com/repos/{LLAMACPP_ROCM_STABLE_REPO}/releases?per_page={LLAMACPP_RELEASES_SEARCHED}" + ); + let body = fetch(&url)?; + let windows = runtime_is_windows(); + newest_llamacpp_tag_with_asset(&body, rocm_version, windows).with_context(|| { + format!( + "none of the {LLAMACPP_RELEASES_SEARCHED} newest {LLAMACPP_ROCM_STABLE_REPO} \ + releases ships {}", + llamacpp_rocm_stable_asset_name("", rocm_version, windows) + ) + }) } /// Read `resources/backend_versions.json`'s pinned `llamacpp.rocm-stable` build tag. @@ -1013,8 +1104,8 @@ mod tests { #[test] fn plain_semver_accepts_release_versions_only() { - assert!(looks_like_plain_semver("10.0.0")); - assert!(looks_like_plain_semver("7.13.0")); + assert!(looks_like_plain_semver("7.14.0")); + assert!(looks_like_plain_semver("10.1.0")); assert!(!looks_like_plain_semver("7.14.0a20260601")); assert!(!looks_like_plain_semver("10.0")); assert!(!looks_like_plain_semver("10.0.0.1")); @@ -1028,8 +1119,8 @@ mod tests { fs::write( &path, serde_json::to_vec_pretty(&json!({ - "llamacpp": { "rocm-stable": "b9752" }, - "therock": { "version": "7.13.0", "architectures": ["gfx1151"] }, + "llamacpp": { "rocm-stable": "b11580" }, + "therock": { "version": "10.1.0", "architectures": ["gfx1151"] }, })) .unwrap(), ) @@ -1037,18 +1128,18 @@ mod tests { assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()) + Some("10.1.0".to_owned()) ); - write_backend_versions_therock_version(&path, "10.0.0").unwrap(); + write_backend_versions_therock_version(&path, "7.14.0").unwrap(); assert_eq!( read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) + Some("7.14.0".to_owned()) ); // Every other key, including sibling keys under `therock`, is preserved. let value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - assert_eq!(value["llamacpp"]["rocm-stable"], "b9752"); + assert_eq!(value["llamacpp"]["rocm-stable"], "b11580"); assert_eq!(value["therock"]["architectures"][0], "gfx1151"); fs::remove_dir_all(&dir).ok(); @@ -1065,7 +1156,7 @@ mod tests { .unwrap(); assert_eq!(read_backend_versions_therock_version(&path), None); - assert!(write_backend_versions_therock_version(&path, "10.0.0").is_err()); + assert!(write_backend_versions_therock_version(&path, "7.14.0").is_err()); fs::remove_dir_all(&dir).ok(); } @@ -1077,8 +1168,8 @@ mod tests { fs::write( &path, serde_json::to_vec_pretty(&json!({ - "llamacpp": { "rocm-stable": "b9752", "vulkan": "b9747" }, - "therock": { "version": "7.13.0" }, + "llamacpp": { "rocm-stable": "b11580", "vulkan": "b11393" }, + "therock": { "version": "10.1.0" }, })) .unwrap(), ) @@ -1086,20 +1177,20 @@ mod tests { assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()) + Some("b11580".to_owned()) ); - write_backend_versions_llamacpp_tag(&path, "b10952").unwrap(); + write_backend_versions_llamacpp_tag(&path, "b12003").unwrap(); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b10952".to_owned()) + Some("b12003".to_owned()) ); // Sibling keys, including the unrelated `llamacpp.vulkan` tag and the // whole `therock` object, are preserved. let value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - assert_eq!(value["llamacpp"]["vulkan"], "b9747"); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["llamacpp"]["vulkan"], "b11393"); + assert_eq!(value["therock"]["version"], "10.1.0"); fs::remove_dir_all(&dir).ok(); } @@ -1115,7 +1206,7 @@ mod tests { .unwrap(); assert_eq!(read_backend_versions_llamacpp_tag(&path), None); - assert!(write_backend_versions_llamacpp_tag(&path, "b10952").is_err()); + assert!(write_backend_versions_llamacpp_tag(&path, "b12003").is_err()); fs::remove_dir_all(&dir).ok(); } @@ -1287,7 +1378,7 @@ mod tests { #[test] fn align_honors_the_disabled_flag_without_touching_the_pin_or_the_injected_steps() { // This pins `align`'s own half of the opt-out contract in isolation: the pin - // and llama.cpp tag stay byte-identical and `align`/`latest_tag` are never + // and llama.cpp tag stay byte-identical and `align`/`newest_tag_with_asset` are never // called. The real production gate -- whether this function is even reached // with `disabled == true` in the first place -- lives one level up, in // `prepare_llamacpp_backend_for_active_rocm_impl`; see @@ -1295,14 +1386,14 @@ mod tests { // below for that half. let dir = scratch_dir("align-disabled"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", true, |_manifest, _target, _force_reinstall, _label| { panic!("disabled means no alignment attempt of any kind") @@ -1314,18 +1405,18 @@ mod tests { ); Ok(()) }, - || panic!("disabled means tier 2's tag lookup must not run either"), + |_target| panic!("disabled means tier 2's tag lookup must not run either"), ); assert_eq!(result.unwrap(), None); assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()), + Some("10.1.0".to_owned()), "the packaged pin must survive untouched when alignment is disabled" ); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()), + Some("b11580".to_owned()), "the packaged llama.cpp tag must survive untouched when alignment is disabled" ); fs::remove_dir_all(&dir).ok(); @@ -1411,8 +1502,8 @@ mod tests { &mut manifest, false, false, - || Ok(Some("10.0.0".to_owned())), - |_path| Some("7.13.0".to_owned()), + || Ok(Some("7.14.0".to_owned())), + |_path| Some("10.1.0".to_owned()), |_manifest| Err(anyhow!("lemond failed to start")), |_manifest, force_reinstall| { assert!( @@ -1894,14 +1985,14 @@ mod tests { fn align_tier1_success_keeps_target_version_pinned() { let dir = scratch_dir("align-tier1-success"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, force_reinstall, _label| { // A stale backend installed against the OLD pin must not be able to @@ -1911,30 +2002,232 @@ mod tests { true }, |_manifest, _force_reinstall| panic!("tier 1 succeeded; no fallback install"), - || panic!("tier 1 succeeded; tier 2's tag lookup must not run"), + |_target| panic!("tier 1 succeeded; tier 2's tag lookup must not run"), ); - assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); + assert_eq!(result.unwrap(), Some("7.14.0".to_owned())); assert_eq!( read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) + Some("7.14.0".to_owned()) ); fs::remove_dir_all(&dir).ok(); } + /// One `GET /repos/lemonade-sdk/llama.cpp/releases` entry, trimmed to the + /// fields Tier 2 reads. + fn llamacpp_release(tag: &str, published_at: &str, assets: &[&str]) -> Value { + json!({ + "tag_name": tag, + "draft": false, + "prerelease": false, + "published_at": published_at, + "assets": assets + .iter() + .map(|name| json!({ "name": name, "state": "uploaded" })) + .collect::>(), + }) + } + + /// Synthetic, modelled on the real listing in early November 2026, where + /// b12062 carries only qualification reports and b11997 no ROCm 7.14 + /// archive. The real b12003 ships both platforms; here it ships Linux + /// archives only, so Linux and Windows pick different builds. + fn llamacpp_releases_fixture() -> Value { + json!([ + llamacpp_release( + "b12062", + "2026-11-03T03:41:03Z", + &[ + "qualification-llama-b12062-bin-ubuntu-rocm-10.1-x64.json", + "qualification-llama-b12062-bin-ubuntu-rocm-7.14-x64.json", + ], + ), + llamacpp_release( + "b12003", + "2026-10-30T03:47:50Z", + &[ + "llama-b12003-bin-ubuntu-rocm-10.1-x64.tar.gz", + "llama-b12003-bin-ubuntu-rocm-7.14-x64.tar.gz", + ], + ), + llamacpp_release( + "b11997", + "2026-10-29T16:11:35Z", + &[ + "llama-b11997-bin-ubuntu-rocm-10.1-x64.tar.gz", + "llama-b11997-bin-win-rocm-10.1-x64.zip", + "qualification-llama-b11997-bin-ubuntu-rocm-7.14-x64.json", + ], + ), + llamacpp_release( + "b11972", + "2026-10-29T00:43:52Z", + &[ + "llama-b11972-bin-ubuntu-rocm-10.1-x64.tar.gz", + "llama-b11972-bin-ubuntu-rocm-7.14-x64.tar.gz", + "llama-b11972-bin-win-rocm-10.1-x64.zip", + "llama-b11972-bin-win-rocm-7.14-x64.zip", + ], + ), + ]) + } + + #[test] + fn llamacpp_asset_name_matches_what_lemonade_downloads() { + assert_eq!( + llamacpp_rocm_stable_asset_name("b11580", "10.1.0", false), + "llama-b11580-bin-ubuntu-rocm-10.1-x64.tar.gz" + ); + assert_eq!( + llamacpp_rocm_stable_asset_name("b11580", "7.14.0", true), + "llama-b11580-bin-win-rocm-7.14-x64.zip" + ); + } + + #[test] + fn tier2_skips_a_newer_release_that_does_not_ship_the_archive() { + let releases = llamacpp_releases_fixture(); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "10.1.0", false).as_deref(), + Some("b12003"), + "the newest release ships only qualification reports" + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "10.1.0", true).as_deref(), + Some("b11997"), + "b12003 ships no Windows archive" + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "7.14.0", false).as_deref(), + Some("b12003") + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "7.14.0", true).as_deref(), + Some("b11972"), + "b12003 ships no Windows archive and b11997 no ROCm 7.14 archive" + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "7.13.0", false), + None, + "no release ships ROCm 7.13" + ); + } + + #[test] + fn tier2_picks_by_publication_date_and_skips_unfinished_releases() { + let mut draft = llamacpp_release( + "b12100", + "2026-11-04T00:00:00Z", + &["llama-b12100-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ); + draft["draft"] = json!(true); + let mut prerelease = llamacpp_release( + "b12101", + "2026-11-04T01:00:00Z", + &["llama-b12101-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ); + prerelease["prerelease"] = json!(true); + let mut uploading = llamacpp_release( + "b12102", + "2026-11-04T02:00:00Z", + &["llama-b12102-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ); + uploading["assets"][0]["state"] = json!("open"); + // An asset whose upload state is not reported is not known to be complete. + let mut stateless = llamacpp_release( + "b12103", + "2026-11-04T03:00:00Z", + &["llama-b12103-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ); + stateless["assets"][0] + .as_object_mut() + .expect("asset object") + .remove("state"); + // Listed out of date order: the listing's order is not trusted. + let releases = json!([ + llamacpp_release( + "b11900", + "2026-10-26T14:08:13Z", + &["llama-b11900-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ), + draft, + prerelease, + uploading, + stateless, + llamacpp_release( + "b11945", + "2026-10-28T00:47:33Z", + &["llama-b11945-bin-ubuntu-rocm-10.1-x64.tar.gz"], + ), + ]); + assert_eq!( + newest_llamacpp_tag_with_asset(&releases, "10.1.0", false).as_deref(), + Some("b11945") + ); + assert_eq!( + newest_llamacpp_tag_with_asset(&json!({}), "10.1.0", false), + None + ); + } + + #[test] + fn tier2_lookup_asks_for_this_hosts_archive() { + // The newer build ships only the other platform's archive, so only a + // lookup that names this host's platform settles on the older one. + let (this_host, other_host) = if cfg!(windows) { + ( + "llama-b11945-bin-win-rocm-10.1-x64.zip", + "llama-b11960-bin-ubuntu-rocm-10.1-x64.tar.gz", + ) + } else { + ( + "llama-b11945-bin-ubuntu-rocm-10.1-x64.tar.gz", + "llama-b11960-bin-win-rocm-10.1-x64.zip", + ) + }; + let releases = json!([ + llamacpp_release("b11960", "2026-10-28T12:41:07Z", &[other_host]), + llamacpp_release("b11945", "2026-10-28T00:47:33Z", &[this_host]), + ]); + let mut requested = String::new(); + let tag = newest_llamacpp_rocm_stable_tag_via("10.1.0", |url| { + requested = url.to_owned(); + Ok(releases) + }) + .unwrap(); + assert_eq!(tag, "b11945"); + assert_eq!( + requested, + "https://api.github.com/repos/lemonade-sdk/llama.cpp/releases?per_page=30" + ); + + let missing = newest_llamacpp_rocm_stable_tag_via("7.13.0", |_| { + Ok(json!([llamacpp_release( + "b11945", + "2026-10-28T00:47:33Z", + &[this_host] + )])) + }) + .unwrap_err(); + assert!(format!("{missing:#}").contains("rocm-7.13"), "{missing:#}"); + let failed = + newest_llamacpp_rocm_stable_tag_via("10.1.0", |_| Err(anyhow!("offline"))).unwrap_err(); + assert!(format!("{failed:#}").contains("offline"), "{failed:#}"); + } + #[test] fn align_falls_through_to_tier2_and_succeeds() { let dir = scratch_dir("align-tier2-success"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let mut align_calls = 0; let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, force_reinstall, _label| { // Both tiers force a reinstall now, so the mock can no longer use @@ -1948,18 +2241,24 @@ mod tests { align_calls > 1 }, |_manifest, _force_reinstall| panic!("tier 2 succeeded; no fallback install"), - || Ok("b10952".to_owned()), + |target| { + assert_eq!( + target, "7.14.0", + "tier 2 looks for a build that ships the active ROCm version's archive" + ); + Ok("b12003".to_owned()) + }, ); - assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); + assert_eq!(result.unwrap(), Some("7.14.0".to_owned())); assert_eq!(align_calls, 2, "both tiers were attempted"); assert_eq!( read_backend_versions_therock_version(&path), - Some("10.0.0".to_owned()) + Some("7.14.0".to_owned()) ); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b10952".to_owned()), + Some("b12003".to_owned()), "tier 2 pins the newer llama.cpp tag" ); fs::remove_dir_all(&dir).ok(); @@ -1973,31 +2272,31 @@ mod tests { // value rather than staying on Tier 2's newer pin. let dir = scratch_dir("align-revert-restores-tag"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| false, |_manifest, force_reinstall| { assert!(force_reinstall); Ok(()) }, - || Ok("b10952".to_owned()), + |_target| Ok("b12003".to_owned()), ); assert_eq!(result.unwrap(), None); assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()) + Some("10.1.0".to_owned()) ); assert_eq!( read_backend_versions_llamacpp_tag(&path), - Some("b9752".to_owned()), + Some("b11580".to_owned()), "the llama.cpp tag must be restored to its original value, not left on Tier 2's pin" ); fs::remove_dir_all(&dir).ok(); @@ -2015,7 +2314,7 @@ mod tests { &path, serde_json::to_vec_pretty(&json!({ "llamacpp": {}, - "therock": { "version": "7.13.0" }, + "therock": { "version": "10.1.0" }, })) .unwrap(), ) @@ -2025,15 +2324,15 @@ mod tests { let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| false, |_manifest, force_reinstall| { assert!(force_reinstall); Ok(()) }, - || Ok("b10952".to_owned()), + |_target| Ok("b12003".to_owned()), ); assert_eq!(result.unwrap(), None); @@ -2046,18 +2345,18 @@ mod tests { } #[test] - fn align_skips_tier2_reinstall_when_pinned_tag_is_already_latest() { + fn align_skips_tier2_reinstall_when_the_pinned_tag_is_the_one_found() { let dir = scratch_dir("align-tier2-tag-unchanged"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let mut align_calls = 0; let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| { align_calls += 1; @@ -2070,15 +2369,16 @@ mod tests { ); Ok(()) }, - // Already the latest tag: Tier 2's own reinstall must not be attempted. - || Ok("b9752".to_owned()), + // The pinned tag is the build Tier 2 found, which Tier 1 already tried: + // Tier 2's own reinstall must not be attempted. + |_target| Ok("b11580".to_owned()), ); assert_eq!(result.unwrap(), None); assert_eq!(align_calls, 1, "only tier 1 was attempted"); assert_eq!( read_backend_versions_therock_version(&path), - Some("7.13.0".to_owned()), + Some("10.1.0".to_owned()), "reverted to the original pinned version" ); fs::remove_dir_all(&dir).ok(); @@ -2091,7 +2391,7 @@ mod tests { // path, and every sibling warning in this function is best-effort. let dir = scratch_dir("align-revert-write-fails"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let mut fallback_called = false; @@ -2100,8 +2400,8 @@ mod tests { let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| { fs::write( @@ -2116,7 +2416,7 @@ mod tests { assert!(force_reinstall); Ok(()) }, - || bail!("network unavailable"), + |_target| bail!("network unavailable"), ); assert_eq!( @@ -2137,7 +2437,7 @@ mod tests { // the pre-existing backend must come back rather than staying deleted. let dir = scratch_dir("align-revert-fallback-install-fails"); let path = dir.join("backend_versions.json"); - write_backend_versions_fixture(&path, "7.13.0", "b9752"); + write_backend_versions_fixture(&path, "10.1.0", "b11580"); let mut manifest = test_manifest(dir.clone()); let llamacpp = dir.join("bin").join("llamacpp"); let server = platform_binary_name("llama-server"); @@ -2148,15 +2448,15 @@ mod tests { let result = align_llamacpp_backend_to_version( &mut manifest, &path, - "10.0.0", - "7.13.0", + "7.14.0", + "10.1.0", false, |_manifest, _target, _force_reinstall, _label| false, |_manifest, force_reinstall| { assert!(force_reinstall); bail!("network unavailable") }, - || Ok("b10952".to_owned()), + |_target| Ok("b12003".to_owned()), ); assert!( diff --git a/engines/lemonade/src/direct_llama.rs b/engines/lemonade/src/direct_llama.rs index de4a96a2e6..7df3094d65 100644 --- a/engines/lemonade/src/direct_llama.rs +++ b/engines/lemonade/src/direct_llama.rs @@ -485,7 +485,7 @@ const DIRECT_LLAMA_SERVER_BACKENDS: [&str; 4] = ["rocm-stable", "rocm-nightly", /// /// Lemonade's backend installer extracts some llama.cpp releases straight into /// `/`, but others land one level deeper under a build-numbered directory -/// (e.g. `rocm-stable/llama-b9752/llama-server`), so both layouts are checked. +/// (e.g. `rocm-stable/llama-b11580/llama-server`), so both layouts are checked. pub(crate) fn find_llama_server_binary(manifest: &LemonadeInstallManifest) -> Option { let llamacpp_dir = manifest.runtime_dir.join("bin").join("llamacpp"); let binary = platform_binary_name("llama-server"); @@ -919,11 +919,11 @@ mod tests { #[test] fn find_llama_server_binary_finds_build_numbered_nesting() { // Some Lemonade llama.cpp releases extract one level deeper than - // `/llama-server`, e.g. `rocm-stable/llama-b9752/llama-server`. + // `/llama-server`, e.g. `rocm-stable/llama-b11580/llama-server`. let dir = scratch_dir("find-server-nested"); let runtime_dir = dir.join("runtime"); let llamacpp = runtime_dir.join("bin").join("llamacpp"); - let nested = llamacpp.join("rocm-stable").join("llama-b9752"); + let nested = llamacpp.join("rocm-stable").join("llama-b11580"); fs::create_dir_all(&nested).unwrap(); let server = nested.join(platform_binary_name("llama-server")); fs::write(&server, b"x").unwrap(); diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index b6617fc215..204c36c3c4 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -37,9 +37,9 @@ const MAX_EMBEDDABLE_SEARCH_DEPTH: usize = 4; /// release has its own digest, so bumping the pin means recording the new pair /// here as well. const EMBEDDABLE_WINDOWS_SHA256: &str = - "50a133bbc35c4f3f8971eafef2c9fe56c4bbbfb0f1032bf8728324d8d8c8a0e1"; + "7ff6c5d6ab612c33690411a7e53d6d0dd62eb515a073375c35015b0778a5a344"; const EMBEDDABLE_LINUX_SHA256: &str = - "bdfd3c3e5d6eda5101c8a32f36e6dd9236ceb9ab2eb66734c32628e6e86e18ac"; + "16f156a7f44a32b20936e76019b24dbccc7b82e60a3acea12e8614da8bca8e90"; /// Embeddable asset token and archive extension for the host this adapter runs /// on. Only `windows-x64` and `ubuntu-x64` are wired up here; the release also @@ -791,6 +791,23 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + #[test] + fn pinned_archive_digests_are_two_distinct_sha256_hex_strings() { + // A digest is pasted by hand on every bump and only a live download + // checks it, so a dropped character or a value pasted twice would + // otherwise pass every offline test. + for digest in [EMBEDDABLE_LINUX_SHA256, EMBEDDABLE_WINDOWS_SHA256] { + assert_eq!(digest.len(), 64, "{digest}"); + assert!( + digest + .chars() + .all(|c| c.is_ascii_digit() || ('a'..='f').contains(&c)), + "{digest}" + ); + } + assert_ne!(EMBEDDABLE_LINUX_SHA256, EMBEDDABLE_WINDOWS_SHA256); + } + #[test] fn a_version_change_forces_re_extraction_over_the_existing_runtime() { let pin = rocm_deps::LEMONADE_VERSION; @@ -816,21 +833,16 @@ mod tests { assert_eq!(installed_runtime_version(runtime_dir), None); - record_runtime_version(runtime_dir, "7.13.0").unwrap(); + record_runtime_version(runtime_dir, "1.2.3").unwrap(); let installed = installed_runtime_version(runtime_dir); - assert_eq!(installed.as_deref(), Some("7.13.0")); + assert_eq!(installed.as_deref(), Some("1.2.3")); assert!(!needs_extraction( false, installed.as_deref(), - "7.13.0", - true - )); - assert!(needs_extraction( - false, - installed.as_deref(), - "7.14.0", + "1.2.3", true )); + assert!(needs_extraction(false, installed.as_deref(), "1.3.0", true)); } #[test] @@ -852,7 +864,7 @@ mod tests { data_dir: env_root.path().join("data"), // no manifest ever written here cache_dir: env_root.path().join("cache"), }; - let version = "7.13.0"; + let version = "1.2.3"; let root = lemonade_root(&paths, Some(env_root.path())); let runtime_dir = runtime_dir_in(&root); fs::create_dir_all(&runtime_dir).unwrap(); @@ -900,7 +912,7 @@ mod tests { data_dir: env_root.path().join("data"), cache_dir: env_root.path().join("cache"), }; - let version = "7.13.0"; + let version = "1.2.3"; // A real, well-formed archive (unlike the stub above): this test // exercises the extraction branch, so `extract_archive` must succeed. @@ -1016,13 +1028,13 @@ mod tests { fs::create_dir_all(&backend).unwrap(); fs::write(backend.join("llama-server"), b"installed backend").unwrap(); - let packaged = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#; + let packaged = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#; let source = dir.join("source-archive"); write_embeddable_with_pins(&source, packaged); let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); let manifest = LemonadeInstallManifest { env_id: "test".to_owned(), - version: "11.5.1".to_owned(), + version: TEST_LEMONADE_VERSION.to_owned(), runtime_dir: runtime_dir.clone(), lemond: lemond_path_in(&runtime_dir), lemonade: lemonade_path_in(&runtime_dir), @@ -1087,12 +1099,20 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// Lemonade version of the restore tests' manifests: synthetic, so a bump + /// of the real pin does not touch these tests. + const TEST_LEMONADE_VERSION: &str = "1.2.3"; + /// Puts `source` where `prepare_embeddable_with` leaves the verified - /// archive of the `11.5.1` test manifests: `/downloads/`. + /// archive of the [`TEST_LEMONADE_VERSION`] test manifests: `/downloads/`. fn cache_archive(root: &Path, source: &Path) { let downloads = root.join("downloads"); fs::create_dir_all(&downloads).unwrap(); - fs::copy(source, downloads.join(embeddable_archive_name("11.5.1"))).unwrap(); + fs::copy( + source, + downloads.join(embeddable_archive_name(TEST_LEMONADE_VERSION)), + ) + .unwrap(); } fn assert_no_pin_temp_files(dir: &Path) { @@ -1109,13 +1129,13 @@ mod tests { } /// Runs the restore against a fake embeddable archive that ships - /// `{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}`, + /// `{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}`, /// with `current` as the pin file already on disk. Returns what the restore /// returned and the pin file's content afterwards. fn restore_over(tag: &str, current: &str) -> (Option, String) { restore_over_packaged( tag, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, current, ) } @@ -1142,7 +1162,7 @@ mod tests { let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); let manifest = LemonadeInstallManifest { env_id: "test".to_owned(), - version: "11.5.1".to_owned(), + version: TEST_LEMONADE_VERSION.to_owned(), runtime_dir: runtime_dir.clone(), lemond: lemond_path_in(&runtime_dir), lemonade: lemonade_path_in(&runtime_dir), @@ -1191,8 +1211,8 @@ mod tests { let (backup, after) = restore_over("restore-keeps-user-edits", current); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); - assert_eq!(value["therock"]["version"], "7.13.0"); - assert_eq!(value["llamacpp"]["rocm-stable"], "b9752"); + assert_eq!(value["therock"]["version"], "10.1.0"); + assert_eq!(value["llamacpp"]["rocm-stable"], "b11580"); assert_eq!(value["llamacpp"]["vulkan"], "b1111", "{after}"); assert_eq!(value["custom"]["x"], 1, "{after}"); assert!(backup.is_some()); @@ -1200,7 +1220,7 @@ mod tests { #[test] fn restore_is_a_no_op_when_only_other_entries_differ_from_the_packaged_file() { - let current = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752","vulkan":"b1111"}}"#; + let current = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580","vulkan":"b1111"}}"#; let (backup, after) = restore_over("restore-noop-other-keys", current); assert!(backup.is_none(), "the alignment pins already match"); @@ -1211,26 +1231,26 @@ mod tests { fn restore_removes_a_pin_the_packaged_file_does_not_have() { // A packaged file without `llamacpp.rocm-stable` must not leave the // kept alignment pin behind; the user's other entries still survive. - let packaged = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"vulkan":"b9747"}}"#; - let current = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b11783","vulkan":"b1111"}}"#; + let packaged = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"vulkan":"b11393"}}"#; + let current = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11783","vulkan":"b1111"}}"#; let (backup, after) = restore_over_packaged("restore-removes-pin", packaged, current); assert!(backup.is_some(), "the kept rocm-stable pin is drift"); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); assert!(value["llamacpp"].get("rocm-stable").is_none(), "{after}"); assert_eq!(value["llamacpp"]["vulkan"], "b1111", "{after}"); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["therock"]["version"], "10.1.0"); } #[test] fn restore_happens_when_only_the_rocm_stable_pin_drifted() { - let current = r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b11783"}}"#; + let current = r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11783"}}"#; let (backup, after) = restore_over("restore-only-rocm-stable", current); assert!(backup.is_some(), "a drifted llama.cpp tag alone is drift"); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); - assert_eq!(value["llamacpp"]["rocm-stable"], "b9752"); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["llamacpp"]["rocm-stable"], "b11580"); + assert_eq!(value["therock"]["version"], "10.1.0"); } #[test] @@ -1239,7 +1259,7 @@ mod tests { assert!(backup.is_some()); let value: serde_json::Value = serde_json::from_str(&after).unwrap(); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["therock"]["version"], "10.1.0"); } /// A runtime tree whose pin file holds a stale kept pin, and the manifest @@ -1257,7 +1277,7 @@ mod tests { .unwrap(); let manifest = LemonadeInstallManifest { env_id: "test".to_owned(), - version: "11.5.1".to_owned(), + version: TEST_LEMONADE_VERSION.to_owned(), runtime_dir: runtime_dir.clone(), lemond: lemond_path_in(&runtime_dir), lemonade: lemonade_path_in(&runtime_dir), @@ -1307,7 +1327,7 @@ mod tests { let source = dir.join("source-archive"); write_embeddable_with_pins( &source, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, ); cache_archive(&root, &source); @@ -1338,7 +1358,7 @@ mod tests { let source = dir.join("source-archive"); write_embeddable_with_pins( &source, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, ); let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); cache_archive(&root, &source); @@ -1362,7 +1382,7 @@ mod tests { ); let value: serde_json::Value = serde_json::from_str(&fs::read_to_string(&pins).unwrap()).unwrap(); - assert_eq!(value["therock"]["version"], "7.13.0"); + assert_eq!(value["therock"]["version"], "10.1.0"); assert_eq!( value["custom"]["x"], 1, "the concurrent writer's other entries survive" @@ -1384,7 +1404,7 @@ mod tests { let source = dir.join("source-archive"); write_embeddable_with_pins( &source, - r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + r#"{"therock":{"version":"10.1.0"},"llamacpp":{"rocm-stable":"b11580"}}"#, ); let sha256 = format!("{:x}", Sha256::digest(fs::read(&source).unwrap())); cache_archive(&root, &source); diff --git a/engines/lemonade/src/llamacpp_backend.rs b/engines/lemonade/src/llamacpp_backend.rs index 5674afe21b..18b037f09c 100644 --- a/engines/lemonade/src/llamacpp_backend.rs +++ b/engines/lemonade/src/llamacpp_backend.rs @@ -140,19 +140,44 @@ pub(crate) fn ensure_best_llamacpp_backend( /// Parse `lemonade backends` table output into `(backend, status)` pairs for the /// `llamacpp` recipe only. Status is one of `installed`/`installable`/`unsupported`. +/// +/// Lemonade also reports two update states, read here as the state they act +/// like: `update_available` (an installed build that still runs, with a newer +/// release out) as `installed`, and `update_required` (an installed build +/// Lemonade will not run until it is reinstalled, e.g. after the pin in +/// `backend_versions.json` moved) as `installable`, so it is reinstalled rather +/// than reused or dropped. The other state Lemonade defines, `action_required` +/// (used by other recipes), leaves the row out. +/// +/// A row that starts in the first column names its recipe; an indented row +/// continues the recipe above it with another backend. The recipe is read from +/// the layout rather than matched against a list of known recipe names: each +/// Lemonade release adds recipes (`llamacpp-hrx`, `ds4`, `thenoise` in +/// 2026.44), and a row for a recipe missing from such a list used to be read as +/// one more `llamacpp` backend, along with every indented row below it. fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { - const RECIPES: [&str; 8] = [ - "flm", - "kokoro", - "llamacpp", - "ryzenai-llm", - "sd-cpp", - "vllm", - "whispercpp", - "embeddings", - ]; - const STATUSES: [&str; 3] = ["installed", "installable", "unsupported"]; - let mut current_recipe = ""; + fn is_state(token: &str) -> bool { + token.starts_with("update_available") + || matches!( + token, + "installed" | "installable" | "update_required" | "unsupported" | "action_required" + ) + } + fn usable_status(token: &str) -> Option<&'static str> { + // Lemonade pads the status column to 16 characters with no separator + // after it, so the one 16-character state runs straight into the + // message (`update_availableNewer upstream release ...`). + if token.starts_with("update_available") { + return Some("installed"); + } + match token { + "installed" => Some("installed"), + "installable" | "update_required" => Some("installable"), + "unsupported" => Some("unsupported"), + _ => None, + } + } + let mut current_recipe = String::new(); let mut result = Vec::new(); for line in output.lines() { let tokens: Vec<&str> = line.split_whitespace().collect(); @@ -162,25 +187,26 @@ fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { if first == "Recipe" || first.starts_with("---") { continue; } - // A row either starts with a recipe name (recipe + its first backend) or, - // for grouped recipes, with a backend name continuing the previous recipe. - let (backend, rest) = if RECIPES.contains(&first) { - current_recipe = match RECIPES.iter().find(|r| **r == first) { - Some(r) => r, - None => current_recipe, - }; + let continues_recipe = line.starts_with(char::is_whitespace); + let (backend, rest) = if continues_recipe { + (first, &tokens[1..]) + } else { + first.clone_into(&mut current_recipe); match tokens.get(1) { Some(backend) => (*backend, &tokens[2..]), None => continue, } - } else { - (first, &tokens[1..]) }; if current_recipe != LLAMACPP_RECIPE { continue; } - if let Some(status) = rest.iter().find(|t| STATUSES.contains(t)) { - result.push((backend.to_owned(), (*status).to_owned())); + // The first state token is the Status column; a state word later in + // the row is message text. + let Some(state) = rest.iter().find(|token| is_state(token)) else { + continue; + }; + if let Some(status) = usable_status(state) { + result.push((backend.to_owned(), status.to_owned())); } } result @@ -370,6 +396,98 @@ vllm rocm unsupported Requires Linux ); } + /// `lemonade backends` as Lemonade 2026.44.1 prints it on a Linux host with + /// an AMD GPU: a `Tier` and a `Format` column, and recipes (`ds4`, + /// `llamacpp-hrx`, `thenoise`) no earlier release listed. + const LEMONADE_2026_44_BACKENDS_TABLE: &str = "\ +Recipe Backend Tier Format Status Message/Version Action +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +acestep rocm experimental native installable Backend is supported but not installed. lemonade backends install acestep:rocm + vulkan experimental native installable Backend is supported but not installed. lemonade backends install acestep:vulkan +ds4 rocm guest native installable Backend is supported but not installed. lemonade backends install ds4:rocm +flm npu core native installable Backend is supported but not installed. lemonade backends install flm:npu +kokoro cpu core native installable Backend is supported but not installed. lemonade backends install kokoro:cpu +llamacpp cpu core native installable Backend is supported but not installed. lemonade backends install llamacpp:cpu + rocm core native installable Backend is supported but not installed. lemonade backends install llamacpp:rocm + vulkan core native installable Backend is supported but not installed. lemonade backends install llamacpp:vulkan +llamacpp-hrx hrx experimental native installable Backend is supported but not installed. lemonade backends install llamacpp-hrx:hrx +moonshine cpu core python installable Backend is supported but not installed. lemonade backends install moonshine:cpu +onnxruntime cpu experimental native installable Backend is supported but not installed. lemonade backends install onnxruntime:cpu +openmoss vulkan experimental native installable Backend is supported but not installed. lemonade backends install openmoss:vulkan +sd-cpp cpu core native installable Backend is supported but not installed. lemonade backends install sd-cpp:cpu + rocm core native installable Backend is supported but not installed. lemonade backends install sd-cpp:rocm + vulkan core native installable Backend is supported but not installed. lemonade backends install sd-cpp:vulkan +thenoise rocm core python installable Backend is supported but not installed. lemonade backends install thenoise:rocm +thinksound rocm experimental native installable Backend is supported but not installed. lemonade backends install thinksound:rocm + vulkan experimental native installable Backend is supported but not installed. lemonade backends install thinksound:vulkan +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +"; + + #[test] + fn parses_only_llamacpp_rows_from_the_2026_44_table() { + let backends = parse_llamacpp_backend_statuses(LEMONADE_2026_44_BACKENDS_TABLE); + assert_eq!( + backends, + vec![ + ("cpu".to_owned(), "installable".to_owned()), + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "installable".to_owned()), + ] + ); + assert_eq!( + select_best_llamacpp_backend(&backends), + Some(("rocm".to_owned(), false)) + ); + } + + #[test] + fn another_recipes_backend_is_never_read_as_a_llamacpp_backend() { + // Plain `lemonade backends` (no `--all`) leaves unsupported backends + // out, so on a host with no GPU backend for llama.cpp its only row is + // `cpu`. The indented `vulkan` row belongs to `openmoss`; read as a + // `llamacpp` backend, it would select a GPU backend Lemonade does not + // offer for llama.cpp here. + let output = "\ +Recipe Backend Tier Format Status Message/Version Action +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +llamacpp cpu core native installable Backend is supported but not installed. lemonade backends install llamacpp:cpu +openmoss rocm experimental native installable Backend is supported but not installed. lemonade backends install openmoss:rocm + vulkan experimental native installable Backend is supported but not installed. lemonade backends install openmoss:vulkan +"; + let backends = parse_llamacpp_backend_statuses(output); + assert_eq!(backends, vec![("cpu".to_owned(), "installable".to_owned())]); + assert_eq!(select_best_llamacpp_backend(&backends), None); + } + + #[test] + fn update_states_read_as_the_state_they_act_like() { + // `update_required`: the installed build will not run until it is + // reinstalled, so it must be installed again, not reused. An + // `update_available` build still runs and is reused; Lemonade prints + // that 16-character state with no space before the message. An + // `action_required` row is left out, whatever words its message holds. + let output = "\ +Recipe Backend Tier Format Status Message/Version Action +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +llamacpp rocm core native update_required Backend update is required before use. lemonade backends install llamacpp:rocm + vulkan core native update_availableNewer upstream release available: b11400 lemonade backends install llamacpp:vulkan + system core native action_required Use the installed driver instead. - +"; + let backends = parse_llamacpp_backend_statuses(output); + assert_eq!( + backends, + vec![ + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "installed".to_owned()), + ] + ); + assert_eq!( + select_best_llamacpp_backend(&backends), + Some(("rocm".to_owned(), false)), + "an update_required rocm backend is reinstalled, not dropped for vulkan" + ); + } + #[test] fn selects_vulkan_when_rocm_unsupported() { let backends = vec![ diff --git a/runtime-deps.toml b/runtime-deps.toml index 44885660bf..2cddff7480 100644 --- a/runtime-deps.toml +++ b/runtime-deps.toml @@ -3,8 +3,9 @@ # This file is the SINGLE SOURCE OF TRUTH for those pins. Nothing else in the # tree may spell a runtime version literal: archive names, download URLs, and # offline fallbacks are all derived from the values below by the `rocm-deps` -# crate, whose build script turns each entry into a Rust constant. Editing a -# version here is the whole bump. +# crate, whose build script turns each entry into a Rust constant. A bump also +# records the new archives' digests where the adapter verifies them; for +# Lemonade, that is `EMBEDDABLE_*_SHA256` in engines/lemonade/src/install.rs. # # Schema: one `[runtime.]` table per managed runtime, with one or more # string fields. Every field becomes a `pub const _: &str` in @@ -15,4 +16,4 @@ # Lemonade embeddable release, published as # `lemonade-embeddable--.{tar.gz|zip}` under the # `v` tag of the lemonade-sdk/lemonade GitHub releases. -version = "11.5.1" +version = "2026.44.1" diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index e1a375cbe6..620b5062a4 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -311,36 +311,36 @@ Feature: Runtime configuration And the execution section says the consent came from the user's --yes # Lemonade's llama.cpp backend re-pins itself to match the ROCm SDK rocm-cli - # actually installed (Tier 1: point the pinned build at it; Tier 2: fall back - # to the newest build if the pin is too old to have shipped a matching - # ROCm-version asset; revert to the packaged default if neither verifies). - # The unit tests exercise that state machine directly against injected - # install/align steps, but nothing else asserts that `rocm engines install - # lemonade` actually surfaces the outcome to a real user -- this is the one - # part of that path with no other e2e coverage. - # - # `--reinstall` re-extracts the packaged embeddable, resetting - # `backend_versions.json` to its pinned defaults, so this fires - # deterministically even against a shared runtime tree where an earlier - # scenario already left Lemonade's backend aligned (a plain install would - # find nothing left to do and print no alignment line at all). Verified - # against real hardware (Strix Halo, gfx1151): a fresh SDK's version does not - # match Lemonade's packaged pin, Tier 1's install 404s (the pinned build - # predates a matching ROCm-version asset), and Tier 2's newest build - # succeeds -- producing exactly the line this scenario asserts. That held for - # an SDK whose TheRock tarball Lemonade could download; for a 7.14.x SDK Tier 2 - # fails too (the tarball is not published where Lemonade 11.5.1 looks), which is - # why `expectations.toml` carries a Windows row for this scenario (ROCMAI-899). + # actually installed (Tier 1: point the pinned build at it; Tier 2: the newest + # build that ships an archive for that ROCm version; revert to the packaged + # default if neither verifies). The unit tests exercise that state machine + # directly against injected install/align steps, but nothing else asserts that + # `rocm engines install lemonade` ends with a ROCm backend paired with the SDK + # on a real GPU -- this is the one part of that path with no other e2e + # coverage. + # + # Which route a lane takes depends on its SDK. An SDK older than Lemonade's + # packaged pin (ROCm 7.14 against 2026.44.1's 10.1.0) is aligned, and the CLI + # says so; an SDK that already is the packaged pin (ROCm 10.1) needs no + # alignment and gets none; a nightly SDK is never aligned and keeps the + # packaged pin. The Then accepts each route and fails a revert: + # it reads the pin the installed runtime tree holds and compares it with the + # active runtime's version. `--reinstall` re-extracts the packaged embeddable, + # resetting `backend_versions.json` to its pinned defaults, so this does not + # depend on what an earlier scenario left in a shared runtime tree. + # `expectations.toml` carries a Windows xfail row for this scenario + # (ROCMAI-899), written for Lemonade 11.5.1, which could not fetch a 7.14.x + # TheRock tarball; a deterministic XPASS on that lane marks the row stale. # # `@requires-engine:lemonade` because vLLM shares the SDK's own runtime # environment and has no llama.cpp backend to align; `@nightly` for the same # reason as the vLLM torch-alignment scenarios above -- a real managed SDK # and a real backend download, not something to repeat on every PR. @id:runtime-lemonade-backend-alignment-reported @requires-gpu @requires-os:windows @requires-engine:lemonade @nightly - Scenario: runtime-16 - Reinstalling Lemonade reports whether its ROCm backend was aligned + Scenario: runtime-16 - Reinstalling Lemonade pairs its ROCm backend with the active SDK Given a managed runtime is active When the user reinstalls the lemonade engine - Then the CLI reports that Lemonade's ROCm backend was aligned to the active SDK + Then Lemonade's ROCm backend is paired with the active SDK # `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT` is the exit for a hand-edited # `backend_versions.json` -- the alignment runs on every Lemonade install, so diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index 1bf377b496..e16cda4a29 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -516,22 +516,104 @@ async fn user_reinstalls_lemonade_engine(world: &mut E2eWorld) { world.cli_stderr = Some(stderr); } -/// Verified against real hardware (Strix Halo, gfx1151): a fresh managed SDK -/// install's version does not match Lemonade's packaged pin, Tier 1's install -/// 404s (the pinned build predates a ROCm-7.14 asset), and Tier 2's newest -/// build succeeds -- producing exactly this line. This is the one part of the -/// alignment path with no other e2e coverage: the unit tests exercise the -/// Tier 1/Tier 2/revert state machine directly (against injected install/align -/// steps), but nothing else asserts that `rocm engines install lemonade` -/// actually surfaces the outcome to the user. -#[then("the CLI reports that Lemonade's ROCm backend was aligned to the active SDK")] -async fn assert_lemonade_backend_alignment_reported(world: &mut E2eWorld) { +/// The pin Lemonade installed against ends up equal to the active SDK, by +/// whichever route this host takes: +/// +/// - the active SDK differs from Lemonade's packaged `therock.version`, and the +/// install aligns the pin to it and says so ("Aligned Lemonade's ROCm llama.cpp +/// backend ..."), as on an SDK older than the pin (ROCm 7.14 against +/// Lemonade 2026.44.1's 10.1.0, whose pinned llama.cpp build also ships a 7.14 +/// archive); +/// - the active SDK already is the packaged pin (a ROCm 10.1 SDK), so there is +/// nothing to align: no alignment line, no alignment attempt, and the ROCm +/// backend installs as packaged. +/// +/// Either way the pin in the installed `backend_versions.json` is read back and +/// compared with the active runtime's own version from `rocm runtimes list`, and +/// the selected backend must be `rocm`: a failed alignment reverts the pin to the +/// packaged value and fails the comparison, and a message without the matching +/// pin fails it too. The unit tests drive the Tier 1/Tier 2/revert state machine +/// against injected steps; this is the only check that `rocm engines install +/// lemonade` reaches a ROCm backend paired with the SDK on real hardware. +#[then("Lemonade's ROCm backend is paired with the active SDK")] +async fn assert_lemonade_backend_paired_with_active_sdk(world: &mut E2eWorld) { let output = world.cli_output.as_deref().expect("no install output"); + let stderr = world.cli_stderr.as_deref().unwrap_or_default(); assert!( output - .contains("Aligned Lemonade's ROCm llama.cpp backend to match the installed ROCm SDK"), - "expected the install to report the ROCm backend alignment outcome:\n{output}" + .contains("Selected the best supported llama.cpp backend for this GPU: llamacpp:rocm"), + "expected Lemonade's ROCm llama.cpp backend to be selected:\n{output}" ); + let pinned_version = installed_therock_pin(output); + let active_version = active_runtime_version(world); + if !is_plain_release_version(&active_version) { + // A dated (nightly) SDK is left alone by design: no alignment is + // attempted and the packaged pin stays. + assert_eq!( + pinned_version, + packaged_therock_pin(output), + "a nightly SDK ({active_version}) must leave Lemonade's packaged pin in place" + ); + assert!( + !output.contains("Aligned Lemonade's ROCm") + && !stderr.contains("Aligned Lemonade's ROCm"), + "a nightly SDK ({active_version}) was aligned to:\nstdout:\n{output}\nstderr:\n{stderr}" + ); + return; + } + assert_eq!( + pinned_version, active_version, + "the installed backend_versions.json does not pin the active ROCm SDK:\nstdout:\n{output}\nstderr:\n{stderr}" + ); + let reported_aligned = output + .contains("Aligned Lemonade's ROCm llama.cpp backend to match the installed ROCm SDK"); + if !reported_aligned { + // Only right when the packaged pin already was the active SDK: then no + // alignment is attempted at all. + assert!( + !stderr.contains("Aligned Lemonade's ROCm backend to") + && !stderr.contains("could not align Lemonade's ROCm backend to"), + "an alignment ran but the install did not report it:\n{stderr}" + ); + } +} + +/// Whether `version` is a plain `major.minor.patch` release, the only kind +/// Lemonade's backend alignment matches against (a nightly such as +/// `7.14.0a20260601` is left alone). +fn is_plain_release_version(version: &str) -> bool { + let parts: Vec<&str> = version.split('.').collect(); + parts.len() == 3 + && parts + .iter() + .all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit())) +} + +/// `therock.version` in the `resources/backend_versions.json` of the runtime +/// tree the install reports (`env_path:`). +fn installed_therock_pin(output: &str) -> String { + let env_path = output + .lines() + .find_map(|line| line.trim().strip_prefix("env_path: ")) + .unwrap_or_else(|| panic!("no env_path in install output:\n{output}")); + let backend_versions_path = + std::path::Path::new(env_path).join("resources/backend_versions.json"); + let contents = std::fs::read_to_string(&backend_versions_path).unwrap_or_else(|error| { + panic!( + "failed to read {}: {error}", + backend_versions_path.display() + ) + }); + let parsed: serde_json::Value = serde_json::from_str(&contents).unwrap_or_else(|error| { + panic!( + "failed to parse {}: {error}", + backend_versions_path.display() + ) + }); + parsed["therock"]["version"] + .as_str() + .unwrap_or_else(|| panic!("no therock.version in {}", backend_versions_path.display())) + .to_owned() } /// The CLI names the variable whenever it is set, so a user can tell it was @@ -562,9 +644,9 @@ async fn assert_lemonade_backend_alignment_opted_out(world: &mut E2eWorld) { /// (marked with `*`; see `ACTIVE_RUNTIME_MARKER` in `apps/rocm/src/main.rs`). /// /// Read independently of Lemonade's own reporting, on purpose: this is the -/// comparator [`assert_packaged_pin_survives`] uses to prove the pin was not -/// rewritten to match it, so it must not come from anything alignment itself -/// could produce (there is no `ROCm SDK:` line in `rocm version` on this +/// comparator [`assert_lemonade_backend_paired_with_active_sdk`] uses to prove +/// the pin was aligned to it, so it must not come from anything alignment +/// itself could produce (there is no `ROCm SDK:` line in `rocm version` on this /// branch -- that surface is a different, later change). fn active_runtime_version(world: &E2eWorld) -> String { let (stdout, _, _) = crate::run_rocm(world, &["runtimes", "list"]); @@ -582,12 +664,12 @@ fn active_runtime_version(world: &E2eWorld) -> String { /// The packaged pin was not rewritten. /// /// Reads `resources/backend_versions.json` inside the runtime tree the install -/// itself reports (`env_path:`) and compares it against the active runtime's -/// own version (from `rocm runtimes list`, not from anything Lemonade's -/// alignment reports), rather than inferring the outcome from an absent log -/// line alone. Both checks matter: the revert path can fail to restore the -/// pin (best-effort, matching its sibling warnings) while still printing no -/// "Aligned ..." line, so absence of that line alone cannot distinguish a +/// itself reports (`env_path:`) and compares it with the copy inside the +/// Lemonade archive the install verified, rather than inferring the outcome +/// from an absent log line alone. The absent-line checks matter too: the +/// revert path can fail to restore the pin (best-effort, matching its sibling +/// warnings) while still printing no "Aligned ..." line, so absence of that +/// line alone cannot distinguish a /// failed-restore from an honestly untouched pin -- hence also checking for /// the revert path's own "could not align ... reverting" warning, which does /// fire whenever a restore was attempted. `--reinstall` (the When's own @@ -610,34 +692,79 @@ async fn assert_packaged_pin_survives(world: &mut E2eWorld) { "an alignment attempt ran (and failed) instead of never starting:\n{stderr}" ); + let pinned_version = installed_therock_pin(output); + let packaged_version = packaged_therock_pin(output); + // Where the active SDK already is the packaged pin (a ROCm 10.1 SDK against + // Lemonade 2026.44.1) this cannot tell an untouched pin from one rewritten to + // the same value; the absent alignment and revert lines above are then the + // whole check. + assert_eq!( + pinned_version, packaged_version, + "the pin differs from the one the Lemonade archive ships even though the user opted out" + ); +} + +/// `therock.version` as the Lemonade embeddable archive the install verified +/// ships it, read from the cached archive beside the runtime tree +/// (`/downloads/` next to `/runtime`) with the system `tar`, which +/// reads both the Linux `.tar.gz` and, as bsdtar on Windows, the `.zip`. The +/// archive is the one named by the install's `env_id` +/// (`lemonade-embeddable-`): the downloads directory is never pruned, +/// so a runner that installed an earlier pin still holds that archive too. +fn packaged_therock_pin(output: &str) -> String { let env_path = output .lines() .find_map(|line| line.trim().strip_prefix("env_path: ")) .unwrap_or_else(|| panic!("no env_path in install output:\n{output}")); - let backend_versions_path = - std::path::Path::new(env_path).join("resources/backend_versions.json"); - let contents = std::fs::read_to_string(&backend_versions_path).unwrap_or_else(|error| { - panic!( - "failed to read {}: {error}", - backend_versions_path.display() - ) - }); - let parsed: serde_json::Value = serde_json::from_str(&contents).unwrap_or_else(|error| { - panic!( - "failed to parse {}: {error}", - backend_versions_path.display() - ) - }); - let pinned_version = parsed["therock"]["version"] + let env_id = output + .lines() + .find_map(|line| line.trim().strip_prefix("env_id: ")) + .unwrap_or_else(|| panic!("no env_id in install output:\n{output}")); + let archive_prefix = format!("{env_id}-"); + let downloads = std::path::Path::new(env_path) + .parent() + .unwrap_or_else(|| panic!("env_path {env_path} has no parent")) + .join("downloads"); + let archive = std::fs::read_dir(&downloads) + .unwrap_or_else(|error| panic!("failed to list {}: {error}", downloads.display())) + .filter_map(Result::ok) + .map(|entry| entry.path()) + .find(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + name.starts_with(&archive_prefix) + && (name.ends_with(".tar.gz") || name.ends_with(".zip")) + }) + }) + .unwrap_or_else(|| panic!("no cached {env_id} archive in {}", downloads.display())); + let name = archive + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + let stem = name + .strip_suffix(".tar.gz") + .or_else(|| name.strip_suffix(".zip")) + .unwrap_or(name); + let member = format!("{stem}/resources/backend_versions.json"); + let extracted = std::process::Command::new("tar") + .arg("-xOf") + .arg(&archive) + .arg(&member) + .output() + .unwrap_or_else(|error| panic!("failed to run tar on {}: {error}", archive.display())); + assert!( + extracted.status.success(), + "tar could not read {member} from {}: {}", + archive.display(), + String::from_utf8_lossy(&extracted.stderr) + ); + let parsed: serde_json::Value = serde_json::from_slice(&extracted.stdout) + .unwrap_or_else(|error| panic!("failed to parse {member}: {error}")); + parsed["therock"]["version"] .as_str() - .unwrap_or_else(|| panic!("no therock.version in {}", backend_versions_path.display())); - - let active_version = active_runtime_version(world); - assert_ne!( - pinned_version, active_version, - "the packaged pin was rewritten to the active ROCm SDK version even though the user \ - opted out" - ); + .unwrap_or_else(|| panic!("no therock.version in {member}")) + .to_owned() } /// The engine inventory reports a usable engine runtime. From fb07c903a56f7c06db8cdec8fe06613e5d8cd001 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 3 Nov 2026 17:32:06 +0200 Subject: [PATCH 02/13] test(e2e): compare the Lemonade archive extension ignoring case The runtime-17 step that finds the cached Lemonade archive compared the `.zip` extension with `ends_with`, which clippy's pedantic `case_sensitive_file_extension_comparisons` rejects. The e2e target is excluded from `--all-targets`, so only the Windows job's `cargo clippy -p e2e-cucumber --test e2e` compiles it and caught it. Derive the archive's stem once, comparing either extension ignoring ASCII case, and reuse it for the member path. Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index e16cda4a29..2fcfaf980b 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -725,27 +725,25 @@ fn packaged_therock_pin(output: &str) -> String { .parent() .unwrap_or_else(|| panic!("env_path {env_path} has no parent")) .join("downloads"); - let archive = std::fs::read_dir(&downloads) + // The archive's top-level directory is its file name without the + // extension. rocm-cli names the archive itself (`lemonade_archive_name`), + // always in lower case; the extension is compared ignoring ASCII case, as + // clippy's `case_sensitive_file_extension_comparisons` asks. + let archive_stem = |path: &std::path::Path| -> Option { + let name = path.file_name()?.to_str()?; + [".tar.gz", ".zip"].iter().find_map(|extension| { + let split = name.len().checked_sub(extension.len())?; + let (stem, tail) = (name.get(..split)?, name.get(split..)?); + (tail.eq_ignore_ascii_case(extension) && stem.starts_with(&archive_prefix)) + .then(|| stem.to_owned()) + }) + }; + let (archive, stem) = std::fs::read_dir(&downloads) .unwrap_or_else(|error| panic!("failed to list {}: {error}", downloads.display())) .filter_map(Result::ok) .map(|entry| entry.path()) - .find(|path| { - path.file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| { - name.starts_with(&archive_prefix) - && (name.ends_with(".tar.gz") || name.ends_with(".zip")) - }) - }) + .find_map(|path| archive_stem(&path).map(|stem| (path, stem))) .unwrap_or_else(|| panic!("no cached {env_id} archive in {}", downloads.display())); - let name = archive - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default(); - let stem = name - .strip_suffix(".tar.gz") - .or_else(|| name.strip_suffix(".zip")) - .unwrap_or(name); let member = format!("{stem}/resources/backend_versions.json"); let extracted = std::process::Command::new("tar") .arg("-xOf") From a7c305cd9c03886c87f753bed5cf6f3237ef64f3 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 10 Nov 2026 17:20:39 +0200 Subject: [PATCH 03/13] fix(lemonade): refuse a host too old for the pinned lemond up front A Linux lemond built against a newer glibc or OpenSSL than the host has exits at once with a dynamic-loader error ("version `GLIBC_2.43' not found"). The user saw only "Lemonade server did not become ready", after the archive had been downloaded and extracted; this is how 2026.45.3 failed on the Ubuntu 24.04 GPU lanes. Record what the pinned lemond needs (glibc 2.38 and OpenSSL 3.0.0 for 2026.44.1, read with objdump -T) beside the archive digests, keyed by the Linux digest so a bump that does not re-read them fails a unit test. Read the host's versions as the newest GLIBC_/OPENSSL_ version node its libc.so.6 and libssl.so.3 define, the same thing the loader compares, locating them through the ldconfig cache. On Linux, `rocm engines install lemonade` now refuses such a host before downloading anything, and starting lemond is refused too. Both name the version needed and the version the host has for each short library. A host version that cannot be read is treated as unknown and never refuses. lemonade-installs-07 drives the install refusal through an e2e-test-hooks override of the host versions, and checks the message and that nothing was downloaded. Signed-off-by: Roman Inflianskas --- Cargo.lock | 1 + README.md | 11 +- crates/rocm-core/src/host_gpu.rs | 3 +- crates/rocm-core/src/host_gpu/wsl.rs | 2 +- crates/rocm-core/src/lib.rs | 3 +- docs/architecture.md | 5 +- docs/engine-plugins.md | 13 +- docs/testing.md | 9 + engines/lemonade/Cargo.toml | 7 + engines/lemonade/src/host_libraries.rs | 388 ++++++++++++++++++ engines/lemonade/src/install.rs | 35 ++ engines/lemonade/src/lib.rs | 13 + engines/lemonade/src/process.rs | 153 +++++++ runtime-deps.toml | 4 +- .../features/lemonade_installs.feature | 13 +- .../tests/e2e/lemonade_install_steps.rs | 62 ++- 16 files changed, 709 insertions(+), 13 deletions(-) create mode 100644 engines/lemonade/src/host_libraries.rs diff --git a/Cargo.lock b/Cargo.lock index 86fe95af56..08de80f5d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3888,6 +3888,7 @@ dependencies = [ "clap", "flate2", "libc", + "object", "proptest", "rocm-core", "rocm-deps", diff --git a/README.md b/README.md index 6cd4841af2..2aa9e9c57b 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,13 @@ serving is Linux or WSL2 only (see [docs/vllm.md](docs/vllm.md)). The minimum supported Linux release, native or under WSL2, is Ubuntu 24.04. On other distributions the equivalent requirement is glibc 2.38 with -`GLIBCXX_3.4.32`: that is what the Lemonade engine is linked against, and every -published build of it needs those versions, so there is no older release to fall -back to. Ubuntu 22.04 ships glibc 2.35 and cannot run it; Ubuntu 24.04 provides -glibc 2.39 and `GLIBCXX_3.4.33`. +`GLIBCXX_3.4.32`, and OpenSSL 3.0.0 or newer: that is what the Lemonade engine +is linked against, and every published build of it needs those versions, so +there is no older release to fall back to. Ubuntu 22.04 ships glibc 2.35 and +cannot run it; Ubuntu 24.04 provides glibc 2.39, `GLIBCXX_3.4.33` and OpenSSL +3.0.13. On an older host, `rocm engines install lemonade` stops before +downloading anything and names the glibc or OpenSSL version Lemonade needs and +the one the host has. > [!IMPORTANT] > **Tech Preview:** This software is provided as-is, without warranty or diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index c2ea229b61..20f03ee939 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -83,12 +83,13 @@ use windows_inventory::{ }; #[cfg(target_os = "linux")] use wsl::detect_wsl_windows_display_probe_text; +pub use wsl::ldconfig_cache; #[cfg(test)] use wsl::local_windows_host_driver_probe; pub use wsl::{WSL_ROCDXG_READY, WslHostDriverProbe, is_wsl_host}; pub(crate) use wsl::{ detect_local_windows_host_driver, detect_wsl_host_driver, detect_wsl_summary, is_wsl1_kernel, - ldconfig_cache, ldconfig_lists_librocdxg, + ldconfig_lists_librocdxg, }; use wsl::{detect_wsl_windows_display_gfx_target_fast, wsl_driver_summary}; diff --git a/crates/rocm-core/src/host_gpu/wsl.rs b/crates/rocm-core/src/host_gpu/wsl.rs index c5900a039a..512d4a1ad6 100644 --- a/crates/rocm-core/src/host_gpu/wsl.rs +++ b/crates/rocm-core/src/host_gpu/wsl.rs @@ -193,7 +193,7 @@ pub(crate) fn is_wsl1_kernel(kernel_release: &str) -> bool { /// Deliberately not routed through [`host_path`]: these are programs this /// process executes, not files it reads to describe the machine, and a /// simulated host root holds no working linker to run. -pub(crate) fn ldconfig_cache() -> Option { +pub fn ldconfig_cache() -> Option { for program in ["ldconfig", "/sbin/ldconfig", "/usr/sbin/ldconfig"] { if let Some(text) = capture_optional_command(program, &["-p"]) { return Some(text); diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 36950d0c36..ffb2c0c84a 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -137,6 +137,7 @@ pub use gfx_target::{ normalize_therock_family, }; pub use hardware_root::host_path; +pub use host_gpu::ldconfig_cache; pub use host_gpu::{ DriverSummary, ExamineSummary, HostGpuSummary, WSL_ROCDXG_READY, WindowsGpuListing, WslHostDriverProbe, WslSummary, default_engine_for_host, default_engine_for_platform, @@ -149,7 +150,7 @@ pub use host_gpu::{ use host_gpu::{KfdGpuNode, kfd_gpu_nodes_in}; use host_gpu::{ detect_linux_sysfs_gfx_target, detect_local_windows_host_driver, detect_wsl_host_driver, - detect_wsl_summary, is_wsl1_kernel, ldconfig_cache, ldconfig_lists_librocdxg, + detect_wsl_summary, is_wsl1_kernel, ldconfig_lists_librocdxg, }; pub use jsonl_store::jsonl_lock_path; pub use log_tail::{ diff --git a/docs/architecture.md b/docs/architecture.md index fbd94b8757..48a913328d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -655,9 +655,10 @@ The Lemonade engine adapter (`rocm-engine-lemonade`): installs the embeddable Le | [`device_policy.rs`](../engines/lemonade/src/device_policy.rs) | Device policy and GPU selection: the GPU-required check, `--device`/`--gpu` parsing, the GPU ordinals to pin before a launch, and the device availability `detect` reports. | domain | | [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) | Hugging Face checkpoint and GGUF resolution, and the direct serve path that runs a packaged `llama-server` on a GGUF (`serve_direct_llama_server`). | domain | | [`footprint.rs`](../engines/lemonade/src/footprint.rs) | What a Lemonade install costs on disk: the pre-install notice and free-space check, and the read-only view of Lemonade's install trees and record. | domain | -| [`install.rs`](../engines/lemonade/src/install.rs) | Embeddable-runtime download, extraction orchestration and manifest persistence, plus the `LemonadeRuntime` handle and `resolve_runtime` over the persisted manifest. | domain | +| [`host_libraries.rs`](../engines/lemonade/src/host_libraries.rs) | Whether this host's glibc and OpenSSL are new enough for the pinned Linux `lemond`: reads the newest `GLIBC_*`/`OPENSSL_*` version node the loader cache's `libc.so.6` and `libssl.so.3` define, and refuses before download and before `lemond` starts, naming both versions. | domain | +| [`install.rs`](../engines/lemonade/src/install.rs) | Embeddable-runtime download, extraction orchestration and manifest persistence, the pinned archives' digests with what the Linux `lemond` needs from the host (`LINUX_LEMOND_NEEDS`), plus the `LemonadeRuntime` handle and `resolve_runtime` over the persisted manifest. | domain | | [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs) | llama.cpp backend discovery and selection: asks `lemonade backends` what the host supports, picks and installs the best GPU backend, and finds its `llama-server`. | domain | -| [`process.rs`](../engines/lemonade/src/process.rs) | `lemond` spawn, environment and path setup, readiness and health polling, and the background relaunch of the `serve-http` command. | domain | +| [`process.rs`](../engines/lemonade/src/process.rs) | `lemond` spawn (refused on Linux for a host older than the pinned build needs), environment and path setup, readiness and health polling, and the background relaunch of the `serve-http` command. | domain | | [`runtime_dir.rs`](../engines/lemonade/src/runtime_dir.rs) | `XDG_RUNTIME_DIR` tier 1, 2 and 3 hardening for the spawned child. | domain | | [`state.rs`](../engines/lemonade/src/state.rs) | Service state file I/O and port and endpoint helpers. | domain | | [`test_support.rs`](../engines/lemonade/src/test_support.rs) | Test-only fixtures shared by the crate's module tests. | flat | diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index 58e313eb1e..92dacff958 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -143,7 +143,18 @@ derived from it, so the tree cannot end up holding two different versions of the same runtime. A Lemonade bump also records the new release's archive digests (`EMBEDDABLE_LINUX_SHA256` and `EMBEDDABLE_WINDOWS_SHA256` in [install.rs](../engines/lemonade/src/install.rs)): the downloaded archive is -checked against them, and a digest cannot be derived from the version. +checked against them, and a digest cannot be derived from the version. It also +re-reads what the new Linux `lemond` needs from the host, the newest `GLIBC_*` +and `OPENSSL_*` versions `objdump -T lemond` lists, into `LINUX_LEMOND_NEEDS` +beside those digests; a unit test fails until that record names the new Linux +archive. On Linux, `rocm engines install lemonade` refuses a host whose glibc +or OpenSSL is older than that before downloading anything, and the engine +refuses to start that `lemond` on such a host, naming the version needed and +the version the host has for each library +([host_libraries.rs](../engines/lemonade/src/host_libraries.rs)). Lemonade +2026.44.1, the current pin, needs glibc 2.38 and OpenSSL 3.0.0, which Ubuntu +24.04 meets; 2026.45.3's `lemond` needs glibc 2.43 and OpenSSL 3.2.0, which it +does not. `rocm engines list` shows the exact plugin directories for the current host. The same output is available in the TUI with `/engine`. diff --git a/docs/testing.md b/docs/testing.md index 908068beeb..36084f1faf 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1720,6 +1720,15 @@ Lemonade copies ROCm CLI keeps (ROCMAI-825), all on the mock lane: `rocm/e2e-test-hooks` build `cargo xtask e2e` makes, and check against the disk that nothing was downloaded. That variable also waives serve's no-GPU refusal, as serve-18's failure hook does. +- `lemonade-installs-07` (Linux) reaches the refusal of a host whose glibc + or OpenSSL is older than the pinned `lemond` needs, with glibc 2.17 and + OpenSSL 1.1.1 reported through `ROCM_E2E_LEMONADE_HOST_GLIBC` and + `ROCM_E2E_LEMONADE_HOST_OPENSSL` (same `rocm/e2e-test-hooks` build). It checks + that the message names both versions of each library and that nothing was + downloaded. Refusing to start `lemond` itself on such a host, the version + comparison and the probe that reads the host's `libc.so.6` and `libssl.so.3` + are unit-tested in [host_libraries.rs](../engines/lemonade/src/host_libraries.rs) + and [process.rs](../engines/lemonade/src/process.rs). - `lemonade-installs-03` plants a data-folder copy and a copy inside a runtime and checks `rocm storage report` lists both and flags the unused one. - `lemonade-installs-04` uninstalls that runtime and checks the install record diff --git a/engines/lemonade/Cargo.toml b/engines/lemonade/Cargo.toml index 98b56d0681..65e966f8ac 100644 --- a/engines/lemonade/Cargo.toml +++ b/engines/lemonade/Cargo.toml @@ -37,6 +37,13 @@ ureq = { version = "2.12", features = ["json", "native-certs"] } # extractor's dependency surface small. zip = { version = "9.0", default-features = false, features = ["deflate"] } +[target.'cfg(target_os = "linux")'.dependencies] +# Reads the version nodes the host's libc.so.6 and libssl.so.3 define, to +# refuse a host older than the pinned `lemond` needs (`src/host_libraries.rs`). +# 0.37 with only the ELF reader: `backtrace` already links this version into +# `rocm`, so it adds no crate to the shipped binary. +object = { version = "0.37", default-features = false, features = ["read_core", "elf"] } + [dev-dependencies] rocm-core = { path = "../../crates/rocm-core", features = ["test-support"] } # Archive-extraction property tests; see `crates/rocm-core/src/archive_props.rs`. diff --git a/engines/lemonade/src/host_libraries.rs b/engines/lemonade/src/host_libraries.rs new file mode 100644 index 0000000000..d16f60e1f4 --- /dev/null +++ b/engines/lemonade/src/host_libraries.rs @@ -0,0 +1,388 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Whether this host's glibc and OpenSSL are new enough for the pinned Linux +//! `lemond`, checked before anything is downloaded and before `lemond` starts. +//! +//! Without this check, a `lemond` built against a newer glibc or OpenSSL than +//! the host has exits at once with a dynamic-loader error ("version +//! `GLIBC_2.43' not found"), and the user sees only that the Lemonade server +//! did not become ready. The versions the pinned build needs are recorded next +//! to its archive digest ([`crate::install::LINUX_LEMOND_NEEDS`]); the host's +//! are read from the libraries the dynamic loader would hand `lemond`, as the +//! newest `GLIBC_*` / `OPENSSL_*` version node each one defines, which is the +//! same thing the loader compares. +//! +//! A host version that cannot be read (no `ldconfig`, a library the cache does +//! not list, a file that does not parse) is reported as unknown and never +//! refuses: `lemond`'s own loader error still reaches the startup log tail then, +//! and a false refusal would block a host that runs it fine. + +use std::cmp::Ordering; +use std::fmt; + +use anyhow::{Result, bail}; + +/// A dotted library version (`2.38`, `3.0.0`), compared numerically with +/// missing components read as zero, so `3.0` and `3.0.0` are equal. +#[derive(Clone, Copy, Debug)] +pub(crate) struct LibVersion { + parts: [u32; 4], + len: usize, +} + +impl LibVersion { + /// Parse a dotted numeric version of one to four components. + pub(crate) fn parse(text: &str) -> Option { + let mut parts = [0_u32; 4]; + let mut len = 0; + for piece in text.split('.') { + if len == parts.len() || piece.is_empty() || !piece.bytes().all(|b| b.is_ascii_digit()) + { + return None; + } + parts[len] = piece.parse().ok()?; + len += 1; + } + Some(Self { parts, len }) + } +} + +impl PartialEq for LibVersion { + fn eq(&self, other: &Self) -> bool { + self.parts == other.parts + } +} + +impl Eq for LibVersion {} + +impl PartialOrd for LibVersion { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for LibVersion { + fn cmp(&self, other: &Self) -> Ordering { + self.parts.cmp(&other.parts) + } +} + +impl fmt::Display for LibVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for (index, part) in self.parts[..self.len].iter().enumerate() { + if index > 0 { + f.write_str(".")?; + } + write!(f, "{part}")?; + } + Ok(()) + } +} + +/// What a pinned Linux `lemond` needs from the host: the newest `GLIBC_*` and +/// `OPENSSL_*` symbol versions it links against, as `objdump -T lemond` lists +/// them, for the archive with digest `archive_sha256`. +pub(crate) struct LemondHostNeeds { + /// The Linux embeddable archive these were read from. A unit test holds it + /// equal to the pinned digest, so a bump cannot keep the old needs. + #[cfg_attr( + not(test), + expect(dead_code, reason = "read only by the test that ties it to the pin") + )] + pub(crate) archive_sha256: &'static str, + pub(crate) glibc: &'static str, + pub(crate) openssl: &'static str, +} + +/// The host's glibc and OpenSSL versions, each `None` when it could not be read. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct HostLibraries { + pub(crate) glibc: Option, + pub(crate) openssl: Option, +} + +/// Why this host cannot run the pinned `lemond`, or `None` when it can or when +/// nothing older than needed was found. +/// +/// Names each library that is too old with the version needed and the version +/// the host has, so the message alone says what to upgrade. +pub(crate) fn host_shortfall( + lemonade_version: &str, + needs: &LemondHostNeeds, + host: &HostLibraries, +) -> Option { + let mut short = Vec::new(); + for (name, needed, have) in [ + ("glibc", needs.glibc, host.glibc), + ("OpenSSL", needs.openssl, host.openssl), + ] { + let needed = LibVersion::parse(needed) + .unwrap_or_else(|| panic!("pinned {name} need {needed:?} is not a version")); + if let Some(have) = have + && have < needed + { + short.push(format!( + "{name} {needed} or newer (this host has {name} {have})" + )); + } + } + if short.is_empty() { + return None; + } + Some(format!( + "this host's libraries are too old for the Lemonade {lemonade_version} server: it needs {}", + short.join(" and ") + )) +} + +/// Fail with [`host_shortfall`]'s message plus `outcome`, which says what the +/// refusal left undone ("Nothing was downloaded."). +pub(crate) fn refuse_if_host_too_old( + lemonade_version: &str, + needs: &LemondHostNeeds, + host: &HostLibraries, + outcome: &str, +) -> Result<()> { + if let Some(shortfall) = host_shortfall(lemonade_version, needs, host) { + bail!("{shortfall}. {outcome}"); + } + Ok(()) +} + +/// The newest numeric version among `names` that start with `prefix` +/// (`GLIBC_2.38` with prefix `GLIBC_`). Names such as `GLIBC_PRIVATE` or the +/// library's own soname entry are skipped. +pub(crate) fn newest_version_node<'a>( + names: impl IntoIterator, + prefix: &str, +) -> Option { + names + .into_iter() + .filter_map(|name| std::str::from_utf8(name).ok()) + .filter_map(|name| name.strip_prefix(prefix)) + .filter_map(LibVersion::parse) + .max() +} + +/// The path `ldconfig -p` gives for `soname` on x86-64, the only Linux +/// architecture the embeddable is installed on. +pub(crate) fn x86_64_library_path<'a>(ldconfig_cache: &'a str, soname: &str) -> Option<&'a str> { + ldconfig_cache.lines().find_map(|line| { + let (left, path) = line.split_once("=>")?; + let left = left.trim(); + let (name, flags) = left.split_once(' ')?; + (name == soname && flags.contains("x86-64")).then(|| path.trim()) + }) +} + +/// The version node names an ELF shared library defines (`SHT_GNU_VERDEF`). +#[cfg(target_os = "linux")] +fn defined_version_names(bytes: &[u8]) -> Option> { + use object::read::elf::ElfFile64; + + let file = ElfFile64::::parse(bytes).ok()?; + let endian = file.endian(); + let sections = file.elf_section_table(); + let (mut defs, link) = sections.gnu_verdef(endian, bytes).ok()??; + let strings = sections.strings(endian, bytes, link).ok()?; + let mut names = Vec::new(); + while let Some((_, mut aux)) = defs.next().ok()? { + if let Some(first) = aux.next().ok()? { + names.push(first.name(endian, strings).ok()?); + } + } + Some(names) +} + +/// The newest `prefix` version node the library at `path` defines. +#[cfg(target_os = "linux")] +fn library_version(path: &str, prefix: &str) -> Option { + let bytes = std::fs::read(path).ok()?; + newest_version_node(defined_version_names(&bytes)?, prefix) +} + +/// Debug/test override of what [`probe_host_libraries`] reports, so a +/// black-box scenario reaches the refusal on a host whose libraries are new. +#[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] +const HOST_GLIBC_TEST_ENV: &str = "ROCM_E2E_LEMONADE_HOST_GLIBC"; +#[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] +const HOST_OPENSSL_TEST_ENV: &str = "ROCM_E2E_LEMONADE_HOST_OPENSSL"; + +/// This host's glibc and OpenSSL, as the libraries the loader cache lists for +/// `libc.so.6` and `libssl.so.3` define them. +#[cfg(target_os = "linux")] +pub(crate) fn probe_host_libraries() -> HostLibraries { + #[cfg(feature = "e2e-test-hooks")] + { + let read = |key: &str| { + std::env::var(key) + .ok() + .and_then(|value| LibVersion::parse(&value)) + }; + let (glibc, openssl) = (read(HOST_GLIBC_TEST_ENV), read(HOST_OPENSSL_TEST_ENV)); + if glibc.is_some() || openssl.is_some() { + return HostLibraries { glibc, openssl }; + } + } + let Some(cache) = rocm_core::ldconfig_cache() else { + return HostLibraries::default(); + }; + let probe = |soname: &str, prefix: &str| { + x86_64_library_path(&cache, soname).and_then(|path| library_version(path, prefix)) + }; + HostLibraries { + glibc: probe("libc.so.6", "GLIBC_"), + openssl: probe("libssl.so.3", "OPENSSL_"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const NEEDS: LemondHostNeeds = LemondHostNeeds { + archive_sha256: "", + glibc: "2.38", + openssl: "3.0.0", + }; + + fn version(text: &str) -> Option { + Some(LibVersion::parse(text).unwrap()) + } + + fn host(glibc: Option<&str>, openssl: Option<&str>) -> HostLibraries { + HostLibraries { + glibc: glibc.and_then(LibVersion::parse), + openssl: openssl.and_then(LibVersion::parse), + } + } + + #[test] + fn versions_compare_numerically_and_ignore_trailing_zeros() { + assert!(version("2.9") < version("2.38")); + assert!(version("2.39") < version("2.43")); + assert!(version("3.0.13") < version("3.2.0")); + assert_eq!(version("3.0"), version("3.0.0")); + assert_eq!(LibVersion::parse("2.2.5").unwrap().to_string(), "2.2.5"); + for bad in ["", "2.", ".2", "2.x", "PRIVATE", "1.2.3.4.5", "-1"] { + assert!(LibVersion::parse(bad).is_none(), "{bad:?} parsed"); + } + } + + /// The states the check can be in, one row each: every library new enough + /// (Ubuntu 24.04 against the 2026.44.1 needs, and exactly at the need), one + /// too old, both too old, and unknown host versions, which never refuse. + #[test] + fn shortfall_names_each_library_that_is_too_old_with_both_versions() { + let rows: [(HostLibraries, Option<&str>); 7] = [ + (host(Some("2.39"), Some("3.0.13")), None), + (host(Some("2.38"), Some("3.0.0")), None), + ( + host(Some("2.35"), Some("3.0.2")), + Some("it needs glibc 2.38 or newer (this host has glibc 2.35)"), + ), + ( + host(Some("2.39"), Some("1.1.1")), + Some("it needs OpenSSL 3.0.0 or newer (this host has OpenSSL 1.1.1)"), + ), + ( + host(Some("2.17"), Some("1.1.1")), + Some( + "it needs glibc 2.38 or newer (this host has glibc 2.17) and \ + OpenSSL 3.0.0 or newer (this host has OpenSSL 1.1.1)", + ), + ), + ( + host(None, Some("1.1.1")), + Some("it needs OpenSSL 3.0.0 or newer (this host has OpenSSL 1.1.1)"), + ), + (host(None, None), None), + ]; + for (host, expected) in rows { + let message = host_shortfall("9.9.9", &NEEDS, &host); + match expected { + None => assert_eq!(message, None, "{host:?}"), + Some(expected) => { + let message = message.unwrap_or_else(|| panic!("no refusal for {host:?}")); + assert!( + message.starts_with( + "this host's libraries are too old for the Lemonade 9.9.9 server: " + ), + "{message}" + ); + assert!(message.ends_with(expected), "{message}"); + } + } + } + } + + #[test] + fn refusal_appends_what_it_left_undone_and_passes_a_new_enough_host() { + let error = refuse_if_host_too_old( + "9.9.9", + &NEEDS, + &host(Some("2.35"), None), + "Nothing was downloaded.", + ) + .unwrap_err() + .to_string(); + assert!( + error.ends_with("(this host has glibc 2.35). Nothing was downloaded."), + "{error}" + ); + refuse_if_host_too_old("9.9.9", &NEEDS, &host(Some("2.39"), Some("3.0.13")), "x").unwrap(); + } + + #[test] + fn newest_version_node_skips_non_numeric_and_other_prefixes() { + let names: [&[u8]; 6] = [ + b"libc.so.6", + b"GLIBC_2.2.5", + b"GLIBC_2.39", + b"GLIBC_2.4", + b"GLIBC_PRIVATE", + b"OPENSSL_3.2.0", + ]; + assert_eq!(newest_version_node(names, "GLIBC_"), version("2.39")); + assert_eq!(newest_version_node(names, "OPENSSL_"), version("3.2.0")); + assert_eq!(newest_version_node(names, "GLIBCXX_"), None); + } + + #[test] + fn library_path_is_the_x86_64_entry_for_exactly_that_soname() { + let cache = "4 libs found in cache `/etc/ld.so.cache'\n\ + \tlibssl.so.3 (libc6) => /usr/lib/i386-linux-gnu/libssl.so.3\n\ + \tlibssl.so.3 (libc6,x86-64) => /lib/x86_64-linux-gnu/libssl.so.3\n\ + \tlibssl.so.30 (libc6,x86-64) => /opt/other/libssl.so.30\n\ + \tlibc.so.6 (libc6,x86-64, OS ABI: Linux 3.2.0) => /lib/x86_64-linux-gnu/libc.so.6\n"; + assert_eq!( + x86_64_library_path(cache, "libssl.so.3"), + Some("/lib/x86_64-linux-gnu/libssl.so.3") + ); + assert_eq!( + x86_64_library_path(cache, "libc.so.6"), + Some("/lib/x86_64-linux-gnu/libc.so.6") + ); + assert_eq!(x86_64_library_path(cache, "libcrypto.so.3"), None); + } + + /// The ELF reader against real libraries: this host's glibc defines version + /// nodes from `GLIBC_2.2.5` up, so the newest is at least 2.17, and an + /// OpenSSL 3 `libssl.so.3` defines `OPENSSL_3.0.0` at least. + #[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))] + #[test] + fn the_running_hosts_glibc_defines_a_readable_version() { + let cache = rocm_core::ldconfig_cache().expect("ldconfig -p"); + let path = x86_64_library_path(&cache, "libc.so.6").expect("libc.so.6 listed"); + let found = library_version(path, "GLIBC_").expect("GLIBC_ version nodes"); + assert!(found >= LibVersion::parse("2.17").unwrap(), "{found}"); + // OpenSSL 3 is optional on a test host; where the cache lists it, its + // own version nodes start at `OPENSSL_3.0.0`. + if let Some(path) = x86_64_library_path(&cache, "libssl.so.3") { + let found = library_version(path, "OPENSSL_").expect("OPENSSL_ version nodes"); + assert!(found >= LibVersion::parse("3.0.0").unwrap(), "{found}"); + } + } +} diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index 204c36c3c4..02f1a34e07 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -41,6 +41,23 @@ const EMBEDDABLE_WINDOWS_SHA256: &str = const EMBEDDABLE_LINUX_SHA256: &str = "16f156a7f44a32b20936e76019b24dbccc7b82e60a3acea12e8614da8bca8e90"; +/// What the `lemond` in the pinned Linux archive needs from the host, read with +/// `objdump -T lemond` (the newest `GLIBC_*` and `OPENSSL_*` versions listed). +/// A host older than either is refused before download and before `lemond` +/// starts; see [`crate::host_libraries`]. Like the digests, these belong to one +/// release: re-read them on every bump and record that archive's digest here, +/// which a unit test holds equal to [`EMBEDDABLE_LINUX_SHA256`]. Lemonade +/// 2026.45.3's `lemond`, for one, needs glibc 2.43 and OpenSSL 3.2.0, which +/// Ubuntu 24.04 does not have; that is why the pin stays at 2026.44.1 +/// (). +#[cfg(any(target_os = "linux", test))] +pub(crate) const LINUX_LEMOND_NEEDS: crate::host_libraries::LemondHostNeeds = + crate::host_libraries::LemondHostNeeds { + archive_sha256: "16f156a7f44a32b20936e76019b24dbccc7b82e60a3acea12e8614da8bca8e90", + glibc: "2.38", + openssl: "3.0.0", + }; + /// Embeddable asset token and archive extension for the host this adapter runs /// on. Only `windows-x64` and `ubuntu-x64` are wired up here; the release also /// publishes `macos-arm64` and `ubuntu-arm64`, which this adapter never selects. @@ -808,6 +825,24 @@ mod tests { assert_ne!(EMBEDDABLE_LINUX_SHA256, EMBEDDABLE_WINDOWS_SHA256); } + /// The host-library needs were read from the archive that is pinned: a bump + /// that records a new Linux digest without re-reading `lemond`'s glibc and + /// OpenSSL needs fails here, instead of refusing hosts by the old needs. + #[test] + fn linux_lemond_needs_belong_to_the_pinned_linux_archive() { + assert_eq!( + LINUX_LEMOND_NEEDS.archive_sha256, EMBEDDABLE_LINUX_SHA256, + "re-read the pinned lemond's needs with `objdump -T lemond` and record \ + them for this archive" + ); + for need in [LINUX_LEMOND_NEEDS.glibc, LINUX_LEMOND_NEEDS.openssl] { + assert!( + crate::host_libraries::LibVersion::parse(need).is_some(), + "{need:?} is not a version" + ); + } + } + #[test] fn a_version_change_forces_re_extraction_over_the_existing_runtime() { let pin = rocm_deps::LEMONADE_VERSION; diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index efa731264c..53a0b4de29 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -29,6 +29,9 @@ mod backend_alignment; mod device_policy; mod direct_llama; mod footprint; +// Linux only: Windows `lemond` has no glibc or OpenSSL floor to check. +#[cfg(any(target_os = "linux", test))] +mod host_libraries; mod install; mod llamacpp_backend; mod process; @@ -411,6 +414,16 @@ fn install_response(request: InstallRequest) -> Result { .env_root .as_deref() .map(normalize_runtime_path_for_host); + // Refused before anything is downloaded: a `lemond` built against a newer + // glibc or OpenSSL than the host has exits at once with a loader error, + // which used to reach the user only as "did not become ready". + #[cfg(target_os = "linux")] + crate::host_libraries::refuse_if_host_too_old( + rocm_deps::LEMONADE_VERSION, + &crate::install::LINUX_LEMOND_NEEDS, + &crate::host_libraries::probe_host_libraries(), + "Nothing was downloaded.", + )?; // Announced and space-checked before anything is downloaded: a fresh tree // is a multi-gigabyte install, and `rocm serve` reaches this on its own. if let Some(preflight) = crate::footprint::plan_install_preflight( diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs index 6d96eed836..a8c6b8987d 100644 --- a/engines/lemonade/src/process.rs +++ b/engines/lemonade/src/process.rs @@ -53,6 +53,64 @@ pub(crate) fn spawn_lemond( port: u16, log_path: Option<&Path>, process_env: &LemonadeProcessEnvironment, +) -> Result { + #[cfg(target_os = "linux")] + return spawn_lemond_on( + &crate::host_libraries::probe_host_libraries(), + manifest, + host, + port, + log_path, + process_env, + ); + #[cfg(not(target_os = "linux"))] + spawn_lemond_unchecked(manifest, host, port, log_path, process_env) +} + +/// [`spawn_lemond`] on Linux, with the host's libraries given so a test can +/// drive the refusal on a host whose libraries are new. +#[cfg(target_os = "linux")] +fn spawn_lemond_on( + host_libraries: &crate::host_libraries::HostLibraries, + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, +) -> Result { + refuse_lemond_on_too_old_host(manifest, host_libraries)?; + spawn_lemond_unchecked(manifest, host, port, log_path, process_env) +} + +/// Refuse to start the pinned Linux `lemond` on a host whose glibc or OpenSSL +/// is older than it needs (see [`crate::host_libraries`]), rather than let it +/// exit with a loader error that reads as a readiness timeout. +/// +/// Applies only to a tree of the pinned version: the recorded needs belong to +/// that archive, and a tree left by an older pin is replaced by the next +/// install, which runs the same check before downloading. +#[cfg(target_os = "linux")] +fn refuse_lemond_on_too_old_host( + manifest: &LemonadeInstallManifest, + host_libraries: &crate::host_libraries::HostLibraries, +) -> Result<()> { + if manifest.version != rocm_deps::LEMONADE_VERSION { + return Ok(()); + } + crate::host_libraries::refuse_if_host_too_old( + &manifest.version, + &crate::install::LINUX_LEMOND_NEEDS, + host_libraries, + "Lemonade was not started.", + ) +} + +fn spawn_lemond_unchecked( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + log_path: Option<&Path>, + process_env: &LemonadeProcessEnvironment, ) -> Result { #[cfg(windows)] if let Some(log_path) = log_path { @@ -720,6 +778,101 @@ mod tests { } } + /// A `lemond` stand-in that records it was started, in a fresh directory. + #[cfg(target_os = "linux")] + fn marker_lemond(version: &str) -> (tempfile::TempDir, LemonadeInstallManifest, PathBuf) { + use std::os::unix::fs::PermissionsExt as _; + let dir = tempfile::tempdir().unwrap(); + let marker = dir.path().join("started"); + let lemond = dir.path().join("lemond"); + fs::write( + &lemond, + format!("#!/bin/sh\ntouch '{}'\n", marker.display()), + ) + .unwrap(); + fs::set_permissions(&lemond, fs::Permissions::from_mode(0o755)).unwrap(); + let manifest = LemonadeInstallManifest { + version: version.to_owned(), + runtime_dir: dir.path().to_path_buf(), + lemond, + ..test_manifest() + }; + (dir, manifest, marker) + } + + /// Start the stand-in through `spawn_lemond`'s Linux path, with the host's + /// libraries given rather than probed. + #[cfg(target_os = "linux")] + fn start_on( + manifest: &LemonadeInstallManifest, + host_libraries: &crate::host_libraries::HostLibraries, + ) -> Result<()> { + let mut child = spawn_lemond_on( + host_libraries, + manifest, + "127.0.0.1", + 1, + None, + &LemonadeProcessEnvironment::default(), + )?; + child.wait()?; + Ok(()) + } + + #[cfg(target_os = "linux")] + fn libraries(glibc: &str, openssl: &str) -> crate::host_libraries::HostLibraries { + crate::host_libraries::HostLibraries { + glibc: crate::host_libraries::LibVersion::parse(glibc), + openssl: crate::host_libraries::LibVersion::parse(openssl), + } + } + + /// On a host older than the pinned `lemond` needs, the refusal names both + /// versions of each short library and says `lemond` was not started, and + /// it really was not: the stand-in never ran. + #[cfg(target_os = "linux")] + #[test] + fn the_pinned_lemond_is_not_started_on_a_host_with_too_old_libraries() { + let (_dir, manifest, marker) = marker_lemond(rocm_deps::LEMONADE_VERSION); + let error = start_on(&manifest, &libraries("2.17", "1.1.1")) + .expect_err("a host older than the pinned lemond needs is refused") + .to_string(); + let needs = &crate::install::LINUX_LEMOND_NEEDS; + for expected in [ + format!("glibc {} or newer (this host has glibc 2.17)", needs.glibc), + format!( + "OpenSSL {} or newer (this host has OpenSSL 1.1.1)", + needs.openssl + ), + format!("the Lemonade {} server", rocm_deps::LEMONADE_VERSION), + ] { + assert!(error.contains(&expected), "missing `{expected}`: {error}"); + } + assert!(error.ends_with("Lemonade was not started."), "{error}"); + assert!(!marker.exists(), "lemond ran despite the refusal"); + } + + /// The other rows: a host that meets the needs starts `lemond`, and so does + /// a tree of another version, whose needs this pin does not record. + #[cfg(target_os = "linux")] + #[test] + fn lemond_starts_on_a_new_enough_host_and_for_a_tree_of_another_version() { + let needs = &crate::install::LINUX_LEMOND_NEEDS; + let (_dir, manifest, marker) = marker_lemond(rocm_deps::LEMONADE_VERSION); + start_on(&manifest, &libraries(needs.glibc, needs.openssl)).unwrap(); + assert!( + marker.exists(), + "lemond did not start on a host meeting its needs" + ); + + let (_dir, manifest, marker) = marker_lemond("0.0.1"); + start_on(&manifest, &libraries("2.17", "1.1.1")).unwrap(); + assert!( + marker.exists(), + "a tree of another version was refused by this pin's needs" + ); + } + /// Build the real backend-install command for a console state and report /// whether the window was hidden, along with the command itself. fn backend_install_for(consoles: OutputConsoles) -> (ProcessCommand, bool) { diff --git a/runtime-deps.toml b/runtime-deps.toml index 2cddff7480..54fbb39814 100644 --- a/runtime-deps.toml +++ b/runtime-deps.toml @@ -5,7 +5,9 @@ # offline fallbacks are all derived from the values below by the `rocm-deps` # crate, whose build script turns each entry into a Rust constant. A bump also # records the new archives' digests where the adapter verifies them; for -# Lemonade, that is `EMBEDDABLE_*_SHA256` in engines/lemonade/src/install.rs. +# Lemonade, that is `EMBEDDABLE_*_SHA256` in engines/lemonade/src/install.rs, +# together with what the new Linux `lemond` needs from the host +# (`LINUX_LEMOND_NEEDS` beside them). # # Schema: one `[runtime.]` table per managed runtime, with one or more # string fields. Every field becomes a `pub const _: &str` in diff --git a/tests/e2e-cucumber/features/lemonade_installs.feature b/tests/e2e-cucumber/features/lemonade_installs.feature index e55ad7fe16..d4d55a3f82 100644 --- a/tests/e2e-cucumber/features/lemonade_installs.feature +++ b/tests/e2e-cucumber/features/lemonade_installs.feature @@ -16,7 +16,8 @@ Feature: Lemonade installs on disk # None downloads anything or needs a GPU, so all run on the mock lane every PR. # lemonade-installs-01 and -02 need the `rocm/e2e-test-hooks` build # `cargo xtask e2e` makes, to report a nearly full drive without filling a - # real one. + # real one; lemonade-installs-07 needs it to report old host libraries on a + # host whose libraries are new. @id:lemonade-installs-refused-without-space Scenario: lemonade-installs-01 - A Lemonade install the drive cannot hold is refused before downloading @@ -62,3 +63,13 @@ Feature: Lemonade installs on disk Then the report does not say that copy is removed with its ROCm install When the user uninstalls the ROCm install holding Lemonade Then that Lemonade copy is still on disk + + # The pinned Linux Lemonade server links against a minimum glibc and OpenSSL. + # On an older host it used to exit with a loader error that reached the user + # only as "Lemonade server did not become ready"; the install now refuses + # first and names both versions. Linux only: Windows has no such floor. + @id:lemonade-installs-refused-on-too-old-host-libraries @requires-os:linux + Scenario: lemonade-installs-07 - A Lemonade install on a host with too old glibc and OpenSSL is refused before downloading + Given this host's glibc and OpenSSL are older than the pinned Lemonade server needs + When the user installs the Lemonade engine + Then the Lemonade install is refused for the host's libraries before anything is downloaded diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs index 10a98f0295..39b6182a98 100644 --- a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -8,7 +8,8 @@ //! Black-box throughout. The two Lemonade trees and the install record are //! planted as plain files in the layout the engine writes, so no scenario //! downloads anything; the refusal scenario reaches the engine's free-space -//! check through the `rocm/e2e-test-hooks` override instead of a full drive. +//! check through the `rocm/e2e-test-hooks` override instead of a full drive, +//! and the host-library refusal through the same build's library override. use std::path::PathBuf; @@ -143,6 +144,65 @@ async fn assert_lemonade_install_refused(world: &mut E2eWorld) { ); } +/// Host library versions older than any Lemonade build needs. Reported through +/// the `rocm/e2e-test-hooks` override, so the refusal is reached on a host +/// whose real glibc and OpenSSL are new. +const OLD_HOST_GLIBC: &str = "2.17"; +const OLD_HOST_OPENSSL: &str = "1.1.1"; + +#[given("this host's glibc and OpenSSL are older than the pinned Lemonade server needs")] +async fn host_libraries_too_old(world: &mut E2eWorld) { + world + .command_env + .push(("ROCM_E2E_LEMONADE_HOST_GLIBC", OLD_HOST_GLIBC.into())); + world + .command_env + .push(("ROCM_E2E_LEMONADE_HOST_OPENSSL", OLD_HOST_OPENSSL.into())); +} + +/// The refusal names, for each library, the version needed and the version +/// the host has, says nothing was downloaded, and that is true on disk. +#[then("the Lemonade install is refused for the host's libraries before anything is downloaded")] +async fn assert_lemonade_install_refused_for_host_libraries(world: &mut E2eWorld) { + let output = combined_output(world); + // Without the hook the engine sees this host's real libraries and starts a + // real install. Name that cause rather than a baffling mismatch. + assert!( + output.contains("this host's libraries are too old for the Lemonade "), + "no host-library refusal: the binary under test was probably built without \ + the `rocm/e2e-test-hooks` feature, which `cargo xtask e2e` enables:\n{output}" + ); + assert_ne!( + world.cli_rc, + Some(0), + "the install must fail on a host too old for the pinned Lemonade server:\n{output}" + ); + for expected in [ + format!("or newer (this host has glibc {OLD_HOST_GLIBC})"), + format!("or newer (this host has OpenSSL {OLD_HOST_OPENSSL})"), + "it needs glibc ".to_owned(), + "Nothing was downloaded.".to_owned(), + ] { + assert!( + output.contains(&expected), + "missing `{expected}`:\n{output}" + ); + } + let tree = data_folder_tree(world); + assert!( + !tree.join("downloads").exists(), + "an archive was downloaded despite the refusal:\n{output}" + ); + assert!( + !tree.join("runtime").exists(), + "a runtime was extracted despite the refusal:\n{output}" + ); + assert!( + !install_record(world).exists(), + "a refused install must not record Lemonade as installed:\n{output}" + ); +} + #[then("the user is told what the Lemonade install needs and where it would go")] async fn assert_lemonade_install_notice(world: &mut E2eWorld) { let output = combined_output(world); From a6adb6976b01d10262b6a241d2a311a5c50791df Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 11 Nov 2026 20:16:29 +0200 Subject: [PATCH 04/13] fix(lemonade): refuse a host whose loader cache has no libssl.so.3 The OpenSSL half of the host check could not fire. The probe read only libssl.so.3, and every libssl.so.3 defines OPENSSL_3.0.0, which is all the pinned lemond needs. A host with only OpenSSL 1.1 has no libssl.so.3 at all, so the probe reported OpenSSL as unknown, never refused, and the user still got "Lemonade server did not become ready". Each library is now a version, missing, or unknown. It is missing when the ldconfig cache was read and lists libc.so.6 but not that library, and the refusal then says the host's loader cache has no libssl.so.3. A cache that could not be read, one that lists not even libc.so.6, and a library that does not parse still read as unknown and never refuse. The cache reading is now a pure function, unit-tested over planted `ldconfig -p` output, and the probe runs once per process. lemonade-installs-07 now reports a host the probe can actually produce, Ubuntu 20.04-like: glibc 2.31 and no libssl.so.3. The README, the docs and docs/manual-testing.md section 3 describe the same two outcomes, and the README no longer says every build needs glibc 2.38, since 2026.45.3 needs more. The start-time refusal has no scenario, because no lane can reach it: `rocm serve` runs the install check first, a Linux serve with a packaged llama-server never starts lemond, and the mock lane has no GPU for serve to resolve. Signed-off-by: Roman Inflianskas --- README.md | 16 +- docs/architecture.md | 2 +- docs/engine-plugins.md | 10 +- docs/manual-testing.md | 16 ++ docs/testing.md | 21 +- engines/lemonade/src/host_libraries.rs | 251 ++++++++++++++---- engines/lemonade/src/process.rs | 63 +++-- .../features/lemonade_installs.feature | 14 +- .../tests/e2e/lemonade_install_steps.rs | 22 +- 9 files changed, 311 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index 2aa9e9c57b..99c4fcf0ac 100644 --- a/README.md +++ b/README.md @@ -45,13 +45,15 @@ serving is Linux or WSL2 only (see [docs/vllm.md](docs/vllm.md)). The minimum supported Linux release, native or under WSL2, is Ubuntu 24.04. On other distributions the equivalent requirement is glibc 2.38 with -`GLIBCXX_3.4.32`, and OpenSSL 3.0.0 or newer: that is what the Lemonade engine -is linked against, and every published build of it needs those versions, so -there is no older release to fall back to. Ubuntu 22.04 ships glibc 2.35 and -cannot run it; Ubuntu 24.04 provides glibc 2.39, `GLIBCXX_3.4.33` and OpenSSL -3.0.13. On an older host, `rocm engines install lemonade` stops before -downloading anything and names the glibc or OpenSSL version Lemonade needs and -the one the host has. +`GLIBCXX_3.4.32`, and OpenSSL 3.0.0 or newer (`libssl.so.3`): that is what the +Lemonade engine is linked against. Every stable Lemonade release from 11.5.1 +through 2026.44.1, the one rocm-cli pins, needs those versions; the newest, +2026.45.3, needs glibc 2.43 and OpenSSL 3.2.0, which is why it is not pinned. +Ubuntu 22.04 ships glibc 2.35 and cannot run it; Ubuntu 24.04 provides glibc +2.39, `GLIBCXX_3.4.33` and OpenSSL 3.0.13. On an older host, +`rocm engines install lemonade` stops before downloading anything and says what +falls short: the glibc version Lemonade needs and the one the host has, or that +the host has no `libssl.so.3`, the library of OpenSSL 3. > [!IMPORTANT] > **Tech Preview:** This software is provided as-is, without warranty or diff --git a/docs/architecture.md b/docs/architecture.md index 48a913328d..8b7168278d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -655,7 +655,7 @@ The Lemonade engine adapter (`rocm-engine-lemonade`): installs the embeddable Le | [`device_policy.rs`](../engines/lemonade/src/device_policy.rs) | Device policy and GPU selection: the GPU-required check, `--device`/`--gpu` parsing, the GPU ordinals to pin before a launch, and the device availability `detect` reports. | domain | | [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) | Hugging Face checkpoint and GGUF resolution, and the direct serve path that runs a packaged `llama-server` on a GGUF (`serve_direct_llama_server`). | domain | | [`footprint.rs`](../engines/lemonade/src/footprint.rs) | What a Lemonade install costs on disk: the pre-install notice and free-space check, and the read-only view of Lemonade's install trees and record. | domain | -| [`host_libraries.rs`](../engines/lemonade/src/host_libraries.rs) | Whether this host's glibc and OpenSSL are new enough for the pinned Linux `lemond`: reads the newest `GLIBC_*`/`OPENSSL_*` version node the loader cache's `libc.so.6` and `libssl.so.3` define, and refuses before download and before `lemond` starts, naming both versions. | domain | +| [`host_libraries.rs`](../engines/lemonade/src/host_libraries.rs) | Whether this host's glibc and OpenSSL are new enough for the pinned Linux `lemond`: reads the newest `GLIBC_*`/`OPENSSL_*` version node the loader cache's `libc.so.6` and `libssl.so.3` define, or finds that a cache listing `libc.so.6` has no `libssl.so.3`, and refuses before download and before `lemond` starts, naming what is needed and what the host has. | domain | | [`install.rs`](../engines/lemonade/src/install.rs) | Embeddable-runtime download, extraction orchestration and manifest persistence, the pinned archives' digests with what the Linux `lemond` needs from the host (`LINUX_LEMOND_NEEDS`), plus the `LemonadeRuntime` handle and `resolve_runtime` over the persisted manifest. | domain | | [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs) | llama.cpp backend discovery and selection: asks `lemonade backends` what the host supports, picks and installs the best GPU backend, and finds its `llama-server`. | domain | | [`process.rs`](../engines/lemonade/src/process.rs) | `lemond` spawn (refused on Linux for a host older than the pinned build needs), environment and path setup, readiness and health polling, and the background relaunch of the `serve-http` command. | domain | diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index 92dacff958..1f6b377644 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -149,9 +149,13 @@ and `OPENSSL_*` versions `objdump -T lemond` lists, into `LINUX_LEMOND_NEEDS` beside those digests; a unit test fails until that record names the new Linux archive. On Linux, `rocm engines install lemonade` refuses a host whose glibc or OpenSSL is older than that before downloading anything, and the engine -refuses to start that `lemond` on such a host, naming the version needed and -the version the host has for each library -([host_libraries.rs](../engines/lemonade/src/host_libraries.rs)). Lemonade +refuses to start that `lemond` on such a host +([host_libraries.rs](../engines/lemonade/src/host_libraries.rs)). For each +library that falls short, the message names the version needed and what the +host has: its version, read from the library the `ldconfig` cache lists, or +that the cache has no such library at all. A host with only OpenSSL 1.1 has no +`libssl.so.3`, so it reads that way. A host whose cache cannot be read, or +whose library does not parse, is not refused. Lemonade 2026.44.1, the current pin, needs glibc 2.38 and OpenSSL 3.0.0, which Ubuntu 24.04 meets; 2026.45.3's `lemond` needs glibc 2.43 and OpenSSL 3.2.0, which it does not. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index a2eb7dd454..8e98d8c570 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -391,6 +391,22 @@ folder run `rocm engines install lemonade --yes`. Expected result: carries on: Lemonade picks the backend after this point and a smaller one may fit. (Stop it before the download if you do not want one.) +To check the refusal of a Linux or WSL host too old for the pinned Lemonade +server (no download needed): the pinned build needs glibc 2.38 and OpenSSL 3 +(`libssl.so.3`). On a real older host, such as Ubuntu 22.04 (glibc 2.35) or +Ubuntu 20.04 (glibc 2.31, OpenSSL 1.1 only), run +`rocm engines install lemonade --yes` in a fresh data folder. On a newer host, +use a debug build and set `ROCM_E2E_LEMONADE_HOST_GLIBC=2.31` and +`ROCM_E2E_LEMONADE_HOST_OPENSSL=missing` first. Expected result: + +- It fails with `this host's libraries are too old for the Lemonade + server: it needs glibc 2.38 or newer (this host has glibc )`, followed + on a host without OpenSSL 3 by `and OpenSSL 3.0.0 or newer (this host's loader + cache has no libssl.so.3)`, and ends `Nothing was downloaded.`. +- The data folder's `engines/lemonade` has no `downloads` or `runtime` + subfolder, and `rocm storage report` lists no Lemonade install. +- On Ubuntu 24.04 or newer the same command installs Lemonade as usual. + With room to spare, on Windows from an interactive console: install Lemonade with no runtime active, then `rocm install sdk`, then `rocm serve `. Expected result: diff --git a/docs/testing.md b/docs/testing.md index 36084f1faf..525d591c11 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1721,14 +1721,19 @@ Lemonade copies ROCm CLI keeps (ROCMAI-825), all on the mock lane: disk that nothing was downloaded. That variable also waives serve's no-GPU refusal, as serve-18's failure hook does. - `lemonade-installs-07` (Linux) reaches the refusal of a host whose glibc - or OpenSSL is older than the pinned `lemond` needs, with glibc 2.17 and - OpenSSL 1.1.1 reported through `ROCM_E2E_LEMONADE_HOST_GLIBC` and - `ROCM_E2E_LEMONADE_HOST_OPENSSL` (same `rocm/e2e-test-hooks` build). It checks - that the message names both versions of each library and that nothing was - downloaded. Refusing to start `lemond` itself on such a host, the version - comparison and the probe that reads the host's `libc.so.6` and `libssl.so.3` - are unit-tested in [host_libraries.rs](../engines/lemonade/src/host_libraries.rs) - and [process.rs](../engines/lemonade/src/process.rs). + or OpenSSL falls short of what the pinned `lemond` needs. It reports an + Ubuntu 20.04-like host through `ROCM_E2E_LEMONADE_HOST_GLIBC=2.31` and + `ROCM_E2E_LEMONADE_HOST_OPENSSL=missing` (same `rocm/e2e-test-hooks` build): + glibc 2.31, and no `libssl.so.3` in the loader cache because only OpenSSL 1.1 + is installed. It checks that the message names what each library needs and + what the host has, and that nothing was downloaded. The probe itself, + including a planted `ldconfig -p` that lists `libc.so.6` but no + `libssl.so.3`, and refusing to start `lemond` on such a host are unit-tested + in [host_libraries.rs](../engines/lemonade/src/host_libraries.rs) and + [process.rs](../engines/lemonade/src/process.rs). No scenario reaches the + start-time refusal: `rocm serve` runs the install check first, a Linux serve + with a packaged `llama-server` never starts `lemond`, and the mock lane has + no GPU for serve to resolve. - `lemonade-installs-03` plants a data-folder copy and a copy inside a runtime and checks `rocm storage report` lists both and flags the unused one. - `lemonade-installs-04` uninstalls that runtime and checks the install record diff --git a/engines/lemonade/src/host_libraries.rs b/engines/lemonade/src/host_libraries.rs index d16f60e1f4..d07ed307c6 100644 --- a/engines/lemonade/src/host_libraries.rs +++ b/engines/lemonade/src/host_libraries.rs @@ -14,13 +14,22 @@ //! newest `GLIBC_*` / `OPENSSL_*` version node each one defines, which is the //! same thing the loader compares. //! -//! A host version that cannot be read (no `ldconfig`, a library the cache does -//! not list, a file that does not parse) is reported as unknown and never -//! refuses: `lemond`'s own loader error still reaches the startup log tail then, -//! and a false refusal would block a host that runs it fine. +//! Each library is found through the `ldconfig` cache, with one of three +//! results ([`HostLibrary`]): +//! +//! - a version, read from the library the cache lists; +//! - missing: the cache was read and lists `libc.so.6`, but not this library's +//! soname. A host with only OpenSSL 1.1 has no `libssl.so.3`, so this is how +//! such a host reads, and it refuses, since `lemond` cannot load at all; +//! - unknown: no cache could be read, the cache lists not even `libc.so.6`, or +//! the library listed did not parse. Unknown never refuses: `lemond`'s own +//! loader error still reaches the startup log tail then, and a false refusal +//! would block a host that runs it fine. use std::cmp::Ordering; use std::fmt; +#[cfg(target_os = "linux")] +use std::sync::OnceLock; use anyhow::{Result, bail}; @@ -96,36 +105,57 @@ pub(crate) struct LemondHostNeeds { pub(crate) openssl: &'static str, } -/// The host's glibc and OpenSSL versions, each `None` when it could not be read. +/// What the probe found for one library. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) enum HostLibrary { + /// Not read: no loader cache, a cache that lists not even `libc.so.6`, or a + /// library that did not parse. Never refuses. + #[default] + Unknown, + /// The cache was read and lists `libc.so.6`, but not this library's soname. + Missing, + /// The newest version node the library defines. + Version(LibVersion), +} + +/// The host's glibc and OpenSSL, as the probe found them. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) struct HostLibraries { - pub(crate) glibc: Option, - pub(crate) openssl: Option, + pub(crate) glibc: HostLibrary, + pub(crate) openssl: HostLibrary, } +/// The soname `lemond` loads for glibc and for OpenSSL, and the prefix of the +/// version nodes it defines. +const GLIBC_SONAME: &str = "libc.so.6"; +const OPENSSL_SONAME: &str = "libssl.so.3"; + /// Why this host cannot run the pinned `lemond`, or `None` when it can or when -/// nothing older than needed was found. +/// nothing short of the needs was found. /// -/// Names each library that is too old with the version needed and the version -/// the host has, so the message alone says what to upgrade. +/// Names each library that falls short with the version needed and what the +/// host has: its version, or that the loader cache has no library of that +/// soname. So the message alone says what to upgrade. pub(crate) fn host_shortfall( lemonade_version: &str, needs: &LemondHostNeeds, host: &HostLibraries, ) -> Option { let mut short = Vec::new(); - for (name, needed, have) in [ - ("glibc", needs.glibc, host.glibc), - ("OpenSSL", needs.openssl, host.openssl), + for (name, soname, needed, have) in [ + ("glibc", GLIBC_SONAME, needs.glibc, host.glibc), + ("OpenSSL", OPENSSL_SONAME, needs.openssl, host.openssl), ] { let needed = LibVersion::parse(needed) .unwrap_or_else(|| panic!("pinned {name} need {needed:?} is not a version")); - if let Some(have) = have - && have < needed - { - short.push(format!( + match have { + HostLibrary::Version(have) if have < needed => short.push(format!( "{name} {needed} or newer (this host has {name} {have})" - )); + )), + HostLibrary::Missing => short.push(format!( + "{name} {needed} or newer (this host's loader cache has no {soname})" + )), + HostLibrary::Version(_) | HostLibrary::Unknown => {} } } if short.is_empty() { @@ -177,6 +207,29 @@ pub(crate) fn x86_64_library_path<'a>(ldconfig_cache: &'a str, soname: &str) -> }) } +/// Read the host's libraries from `ldconfig -p`'s text (`None` when it could not +/// be run), with `read_version(path, prefix)` reading one library's newest +/// version node. Pure apart from `read_version`, so a test plants the cache. +pub(crate) fn host_libraries_from_cache( + cache: Option<&str>, + read_version: impl Fn(&str, &str) -> Option, +) -> HostLibraries { + let Some(cache) = cache else { + return HostLibraries::default(); + }; + // A cache that lists not even the C library says nothing about the others. + let cache_lists_libc = x86_64_library_path(cache, GLIBC_SONAME).is_some(); + let probe = |soname: &str, prefix: &str| match x86_64_library_path(cache, soname) { + Some(path) => read_version(path, prefix).map_or(HostLibrary::Unknown, HostLibrary::Version), + None if cache_lists_libc => HostLibrary::Missing, + None => HostLibrary::Unknown, + }; + HostLibraries { + glibc: probe(GLIBC_SONAME, "GLIBC_"), + openssl: probe(OPENSSL_SONAME, "OPENSSL_"), + } +} + /// The version node names an ELF shared library defines (`SHT_GNU_VERDEF`). #[cfg(target_os = "linux")] fn defined_version_names(bytes: &[u8]) -> Option> { @@ -205,37 +258,38 @@ fn library_version(path: &str, prefix: &str) -> Option { /// Debug/test override of what [`probe_host_libraries`] reports, so a /// black-box scenario reaches the refusal on a host whose libraries are new. +/// Each takes a version, or `missing` for a library the cache does not list. #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] const HOST_GLIBC_TEST_ENV: &str = "ROCM_E2E_LEMONADE_HOST_GLIBC"; #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] const HOST_OPENSSL_TEST_ENV: &str = "ROCM_E2E_LEMONADE_HOST_OPENSSL"; /// This host's glibc and OpenSSL, as the libraries the loader cache lists for -/// `libc.so.6` and `libssl.so.3` define them. +/// `libc.so.6` and `libssl.so.3` define them. Probed once per process: the +/// install and every `lemond` start ask, and the answer does not change. #[cfg(target_os = "linux")] pub(crate) fn probe_host_libraries() -> HostLibraries { - #[cfg(feature = "e2e-test-hooks")] - { - let read = |key: &str| { - std::env::var(key) - .ok() - .and_then(|value| LibVersion::parse(&value)) - }; - let (glibc, openssl) = (read(HOST_GLIBC_TEST_ENV), read(HOST_OPENSSL_TEST_ENV)); - if glibc.is_some() || openssl.is_some() { - return HostLibraries { glibc, openssl }; + static PROBED: OnceLock = OnceLock::new(); + *PROBED.get_or_init(|| { + #[cfg(feature = "e2e-test-hooks")] + { + let read = |key: &str| match std::env::var(key).ok().as_deref() { + Some("missing") => HostLibrary::Missing, + Some(value) => { + LibVersion::parse(value).map_or(HostLibrary::Unknown, HostLibrary::Version) + } + None => HostLibrary::Unknown, + }; + let hooked = HostLibraries { + glibc: read(HOST_GLIBC_TEST_ENV), + openssl: read(HOST_OPENSSL_TEST_ENV), + }; + if hooked != HostLibraries::default() { + return hooked; + } } - } - let Some(cache) = rocm_core::ldconfig_cache() else { - return HostLibraries::default(); - }; - let probe = |soname: &str, prefix: &str| { - x86_64_library_path(&cache, soname).and_then(|path| library_version(path, prefix)) - }; - HostLibraries { - glibc: probe("libc.so.6", "GLIBC_"), - openssl: probe("libssl.so.3", "OPENSSL_"), - } + host_libraries_from_cache(rocm_core::ldconfig_cache().as_deref(), library_version) + }) } #[cfg(test)] @@ -252,10 +306,20 @@ mod tests { Some(LibVersion::parse(text).unwrap()) } + /// One library as a test writes it: a version, `missing`, or `None` for + /// unknown. + fn library(text: Option<&str>) -> HostLibrary { + match text { + None => HostLibrary::Unknown, + Some("missing") => HostLibrary::Missing, + Some(text) => HostLibrary::Version(LibVersion::parse(text).unwrap()), + } + } + fn host(glibc: Option<&str>, openssl: Option<&str>) -> HostLibraries { HostLibraries { - glibc: glibc.and_then(LibVersion::parse), - openssl: openssl.and_then(LibVersion::parse), + glibc: library(glibc), + openssl: library(openssl), } } @@ -271,11 +335,13 @@ mod tests { } } - /// The states the check can be in, one row each: every library new enough - /// (Ubuntu 24.04 against the 2026.44.1 needs, and exactly at the need), one - /// too old, both too old, and unknown host versions, which never refuse. + /// The states the check can be in, one row each, as the probe can produce + /// them: every library new enough (Ubuntu 24.04 against the 2026.44.1 + /// needs, and exactly at the need); glibc too old (Ubuntu 22.04); OpenSSL 3 + /// missing with glibc too old (Ubuntu 20.04, OpenSSL 1.1 only); OpenSSL 3 + /// missing alone; and unknown versions, which never refuse. #[test] - fn shortfall_names_each_library_that_is_too_old_with_both_versions() { + fn shortfall_names_each_library_that_falls_short_and_what_the_host_has() { let rows: [(HostLibraries, Option<&str>); 7] = [ (host(Some("2.39"), Some("3.0.13")), None), (host(Some("2.38"), Some("3.0.0")), None), @@ -284,20 +350,19 @@ mod tests { Some("it needs glibc 2.38 or newer (this host has glibc 2.35)"), ), ( - host(Some("2.39"), Some("1.1.1")), - Some("it needs OpenSSL 3.0.0 or newer (this host has OpenSSL 1.1.1)"), - ), - ( - host(Some("2.17"), Some("1.1.1")), + host(Some("2.31"), Some("missing")), Some( - "it needs glibc 2.38 or newer (this host has glibc 2.17) and \ - OpenSSL 3.0.0 or newer (this host has OpenSSL 1.1.1)", + "it needs glibc 2.38 or newer (this host has glibc 2.31) and \ + OpenSSL 3.0.0 or newer (this host's loader cache has no libssl.so.3)", ), ), ( - host(None, Some("1.1.1")), - Some("it needs OpenSSL 3.0.0 or newer (this host has OpenSSL 1.1.1)"), + host(Some("2.39"), Some("missing")), + Some( + "it needs OpenSSL 3.0.0 or newer (this host's loader cache has no libssl.so.3)", + ), ), + (host(None, Some("3.0.13")), None), (host(None, None), None), ]; for (host, expected) in rows { @@ -318,6 +383,25 @@ mod tests { } } + /// An OpenSSL 3 older than a pin needs is reported by version: no pin needs + /// that today, but 2026.45.3 needed 3.2.0, which Ubuntu 24.04's 3.0.13 is not. + #[test] + fn an_openssl_3_older_than_the_need_is_named_by_version() { + let needs = LemondHostNeeds { + archive_sha256: "", + glibc: "2.43", + openssl: "3.2.0", + }; + let message = host_shortfall("9.9.9", &needs, &host(Some("2.39"), Some("3.0.13"))).unwrap(); + assert!( + message.ends_with( + "it needs glibc 2.43 or newer (this host has glibc 2.39) and \ + OpenSSL 3.2.0 or newer (this host has OpenSSL 3.0.13)" + ), + "{message}" + ); + } + #[test] fn refusal_appends_what_it_left_undone_and_passes_a_new_enough_host() { let error = refuse_if_host_too_old( @@ -335,6 +419,63 @@ mod tests { refuse_if_host_too_old("9.9.9", &NEEDS, &host(Some("2.39"), Some("3.0.13")), "x").unwrap(); } + const LIBC_LINE: &str = + "\tlibc.so.6 (libc6,x86-64, OS ABI: Linux 3.2.0) => /lib/x86_64-linux-gnu/libc.so.6\n"; + const LIBSSL_LINE: &str = "\tlibssl.so.3 (libc6,x86-64) => /lib/x86_64-linux-gnu/libssl.so.3\n"; + const LIBSSL11_LINE: &str = + "\tlibssl.so.1.1 (libc6,x86-64) => /lib/x86_64-linux-gnu/libssl.so.1.1\n"; + + /// A reader standing in for the ELF read: glibc 2.31 and OpenSSL 3.0.13, + /// or `None` for a library that does not parse. + fn planted_reader(unparseable: &'static str) -> impl Fn(&str, &str) -> Option { + move |path, _prefix| { + if path.ends_with(unparseable) { + None + } else if path.ends_with("libc.so.6") { + LibVersion::parse("2.31") + } else { + LibVersion::parse("3.0.13") + } + } + } + + /// The probe over a planted `ldconfig -p`, one row per state: an OpenSSL + /// 1.1-only host reads OpenSSL as missing and is refused; a host with both + /// reads versions; and an unreadable cache, a cache that lists no C library, + /// and a listed library that does not parse all read unknown. + #[test] + fn a_read_cache_without_libssl_3_reports_openssl_missing_and_refuses() { + let header = "3 libs found in cache `/etc/ld.so.cache'\n"; + let openssl_1_1_only = format!("{header}{LIBC_LINE}{LIBSSL11_LINE}"); + let probed = host_libraries_from_cache(Some(&openssl_1_1_only), planted_reader("-")); + assert_eq!(probed, host(Some("2.31"), Some("missing"))); + let message = host_shortfall("9.9.9", &NEEDS, &probed).expect("refused"); + assert!( + message + .contains("OpenSSL 3.0.0 or newer (this host's loader cache has no libssl.so.3)"), + "{message}" + ); + + let both = format!("{header}{LIBC_LINE}{LIBSSL_LINE}"); + assert_eq!( + host_libraries_from_cache(Some(&both), planted_reader("-")), + host(Some("2.31"), Some("3.0.13")) + ); + assert_eq!( + host_libraries_from_cache(Some(&both), planted_reader("libssl.so.3")), + host(Some("2.31"), None) + ); + assert_eq!( + host_libraries_from_cache(None, planted_reader("-")), + HostLibraries::default() + ); + let no_libc = format!("{header}{LIBSSL11_LINE}"); + assert_eq!( + host_libraries_from_cache(Some(&no_libc), planted_reader("-")), + HostLibraries::default() + ); + } + #[test] fn newest_version_node_skips_non_numeric_and_other_prefixes() { let names: [&[u8]; 6] = [ diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs index a8c6b8987d..f493dc444d 100644 --- a/engines/lemonade/src/process.rs +++ b/engines/lemonade/src/process.rs @@ -802,46 +802,75 @@ mod tests { /// Start the stand-in through `spawn_lemond`'s Linux path, with the host's /// libraries given rather than probed. + /// + /// A script written a moment ago can fail to exec with `ETXTBSY` when + /// another test thread forks while the write handle is still open; that, + /// and only that, is retried. #[cfg(target_os = "linux")] fn start_on( manifest: &LemonadeInstallManifest, host_libraries: &crate::host_libraries::HostLibraries, ) -> Result<()> { - let mut child = spawn_lemond_on( - host_libraries, - manifest, - "127.0.0.1", - 1, - None, - &LemonadeProcessEnvironment::default(), - )?; + let mut attempts = 0; + let mut child = loop { + match spawn_lemond_on( + host_libraries, + manifest, + "127.0.0.1", + 1, + None, + &LemonadeProcessEnvironment::default(), + ) { + Err(error) + if attempts < 50 + && error.chain().any(|cause| { + cause + .downcast_ref::() + .and_then(std::io::Error::raw_os_error) + == Some(libc::ETXTBSY) + }) => + { + attempts += 1; + std::thread::sleep(Duration::from_millis(10)); + } + result => break result?, + } + }; child.wait()?; Ok(()) } + /// Host libraries as a test writes them: a version, or `missing` for a + /// library the loader cache does not list. #[cfg(target_os = "linux")] fn libraries(glibc: &str, openssl: &str) -> crate::host_libraries::HostLibraries { + use crate::host_libraries::{HostLibrary, LibVersion}; + let library = |text: &str| match text { + "missing" => HostLibrary::Missing, + text => HostLibrary::Version(LibVersion::parse(text).unwrap()), + }; crate::host_libraries::HostLibraries { - glibc: crate::host_libraries::LibVersion::parse(glibc), - openssl: crate::host_libraries::LibVersion::parse(openssl), + glibc: library(glibc), + openssl: library(openssl), } } - /// On a host older than the pinned `lemond` needs, the refusal names both - /// versions of each short library and says `lemond` was not started, and - /// it really was not: the stand-in never ran. + /// On a host older than the pinned `lemond` needs (Ubuntu 20.04: glibc 2.31 + /// and OpenSSL 1.1 only), the refusal names what each short library needs + /// and what the host has, says `lemond` was not started, and it really was + /// not: the stand-in never ran. #[cfg(target_os = "linux")] #[test] fn the_pinned_lemond_is_not_started_on_a_host_with_too_old_libraries() { let (_dir, manifest, marker) = marker_lemond(rocm_deps::LEMONADE_VERSION); - let error = start_on(&manifest, &libraries("2.17", "1.1.1")) + let error = start_on(&manifest, &libraries("2.31", "missing")) .expect_err("a host older than the pinned lemond needs is refused") .to_string(); let needs = &crate::install::LINUX_LEMOND_NEEDS; for expected in [ - format!("glibc {} or newer (this host has glibc 2.17)", needs.glibc), + format!("glibc {} or newer (this host has glibc 2.31)", needs.glibc), format!( - "OpenSSL {} or newer (this host has OpenSSL 1.1.1)", + "OpenSSL {} or newer (this host's loader cache has no libssl.so.3)", needs.openssl ), format!("the Lemonade {} server", rocm_deps::LEMONADE_VERSION), @@ -866,7 +895,7 @@ mod tests { ); let (_dir, manifest, marker) = marker_lemond("0.0.1"); - start_on(&manifest, &libraries("2.17", "1.1.1")).unwrap(); + start_on(&manifest, &libraries("2.31", "missing")).unwrap(); assert!( marker.exists(), "a tree of another version was refused by this pin's needs" diff --git a/tests/e2e-cucumber/features/lemonade_installs.feature b/tests/e2e-cucumber/features/lemonade_installs.feature index d4d55a3f82..aba44b175f 100644 --- a/tests/e2e-cucumber/features/lemonade_installs.feature +++ b/tests/e2e-cucumber/features/lemonade_installs.feature @@ -67,9 +67,17 @@ Feature: Lemonade installs on disk # The pinned Linux Lemonade server links against a minimum glibc and OpenSSL. # On an older host it used to exit with a loader error that reached the user # only as "Lemonade server did not become ready"; the install now refuses - # first and names both versions. Linux only: Windows has no such floor. + # first and names, for each library, what it needs and what the host has. The + # host reported is Ubuntu 20.04-like: glibc 2.31, and OpenSSL 1.1 only, so no + # libssl.so.3. Linux only: Windows has no such floor. + # + # The same refusal also guards starting the Lemonade server itself. No + # scenario reaches that one: `rocm serve` runs this install check first, a + # Linux serve with a packaged llama-server never starts the Lemonade server, + # and the mock lane has no GPU for serve to resolve. The unit tests in + # engines/lemonade/src/process.rs cover it. @id:lemonade-installs-refused-on-too-old-host-libraries @requires-os:linux - Scenario: lemonade-installs-07 - A Lemonade install on a host with too old glibc and OpenSSL is refused before downloading - Given this host's glibc and OpenSSL are older than the pinned Lemonade server needs + Scenario: lemonade-installs-07 - A Lemonade install on a host with too old a glibc and no OpenSSL 3 is refused before downloading + Given this host's glibc is older than the pinned Lemonade server needs and it has no OpenSSL 3 When the user installs the Lemonade engine Then the Lemonade install is refused for the host's libraries before anything is downloaded diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs index 39b6182a98..713c2facd9 100644 --- a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -144,13 +144,14 @@ async fn assert_lemonade_install_refused(world: &mut E2eWorld) { ); } -/// Host library versions older than any Lemonade build needs. Reported through -/// the `rocm/e2e-test-hooks` override, so the refusal is reached on a host -/// whose real glibc and OpenSSL are new. -const OLD_HOST_GLIBC: &str = "2.17"; -const OLD_HOST_OPENSSL: &str = "1.1.1"; - -#[given("this host's glibc and OpenSSL are older than the pinned Lemonade server needs")] +/// An Ubuntu 20.04-like host, as the probe reads one: glibc 2.31, and no +/// `libssl.so.3` in the loader cache because it has only OpenSSL 1.1. Reported +/// through the `rocm/e2e-test-hooks` override, so the refusal is reached on a +/// host whose real glibc and OpenSSL are new. +const OLD_HOST_GLIBC: &str = "2.31"; +const OLD_HOST_OPENSSL: &str = "missing"; + +#[given("this host's glibc is older than the pinned Lemonade server needs and it has no OpenSSL 3")] async fn host_libraries_too_old(world: &mut E2eWorld) { world .command_env @@ -160,8 +161,9 @@ async fn host_libraries_too_old(world: &mut E2eWorld) { .push(("ROCM_E2E_LEMONADE_HOST_OPENSSL", OLD_HOST_OPENSSL.into())); } -/// The refusal names, for each library, the version needed and the version -/// the host has, says nothing was downloaded, and that is true on disk. +/// The refusal names, for each library, the version needed and what the host +/// has (a version, or no `libssl.so.3`), says nothing was downloaded, and that +/// is true on disk. #[then("the Lemonade install is refused for the host's libraries before anything is downloaded")] async fn assert_lemonade_install_refused_for_host_libraries(world: &mut E2eWorld) { let output = combined_output(world); @@ -179,7 +181,7 @@ async fn assert_lemonade_install_refused_for_host_libraries(world: &mut E2eWorld ); for expected in [ format!("or newer (this host has glibc {OLD_HOST_GLIBC})"), - format!("or newer (this host has OpenSSL {OLD_HOST_OPENSSL})"), + "or newer (this host's loader cache has no libssl.so.3)".to_owned(), "it needs glibc ".to_owned(), "Nothing was downloaded.".to_owned(), ] { From 202490cad3827419870ed39f66ee996a68cc1473 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 11 Nov 2026 20:16:29 +0200 Subject: [PATCH 05/13] test(e2e): stop calling Lemonade 11.5.1 the pin in two Windows rows The runtime-16 (ROCMAI-899) and serve-06 (ROCMAI-900) Windows rows were written while Lemonade 11.5.1 was pinned, and their reasons still said so. Say they were observed with 11.5.1 and have not been re-checked with 2026.44.1, which fetches TheRock elsewhere and whose llama.cpp build ships a ROCm 7.14 archive. Both scenarios are @nightly, so pull-request lanes have not run them yet; a stale XPASS on the first Windows nightly after this pin lands is the signal to delete the row. Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/expectations.toml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index a50f1229fd..1411c9d8b0 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -218,10 +218,10 @@ reason = "Lemonade managed serve never exposes the model: observed backend argum serve_timeout_secs = 90 flaky = true -# --- Lemonade 11.5.1 cannot align its llama.cpp backend to a ROCm 7.14.x SDK --- +# --- Lemonade 11.5.1 could not align its llama.cpp backend to a ROCm 7.14.x SDK --- # `rocm engines install lemonade` tries to align Lemonade's `llamacpp:rocm` -# backend with the active ROCm SDK. Lemonade 11.5.1, the pin in -# `runtime-deps.toml`, downloads the TheRock tarball it pairs with the backend +# backend with the active ROCm SDK. Lemonade 11.5.1, which `runtime-deps.toml` +# pinned when these rows were written, downloads the TheRock tarball it pairs with the backend # only from https://repo.amd.com/rocm/tarball/, a path the 7.14.x tarballs are not # published to (AMD's 7.14 install guide uses /rocm/tarball-multi-arch/), so the # download answers HTTP 403 for `therock-dist---7.14.1.tar.gz`. The @@ -244,12 +244,19 @@ flaky = true # `serve_timeout_secs`: serve-06's own `@serve-timeout:2400` tag takes precedence # over a row, so a shorter wait here would have no effect, and the failing # serve-06 still holds the Windows lane for the full 2400s wait. +# +# The pin is now Lemonade 2026.44.1, which fetches TheRock from AMD's whl-next +# wheel index or its core tarball location, and whose pinned llama.cpp build +# (b11580) ships a ROCm 7.14 archive. So both rows may now XPASS. That has not +# been observed yet: both scenarios are `@nightly`, which pull-request lanes do +# not run. A stale XPASS on the first Windows nightly run after the 2026.44.1 pin +# lands means the cause is gone, and the row is deleted then. [["runtime-lemonade-backend-alignment-reported"]] when = { effective_engine = "lemonade", os = "windows" } bug = "ROCMAI-899" -reason = "Lemonade 11.5.1 fetches TheRock only from repo.amd.com/rocm/tarball/, where 7.14.x is not published (HTTP 403), so its backend cannot be aligned to a 7.14.x SDK and the install reverts to the pinned 7.13 backend." +reason = "Observed with Lemonade 11.5.1, which fetched TheRock only from repo.amd.com/rocm/tarball/, where 7.14.x is not published (HTTP 403), so its backend could not be aligned to a 7.14.x SDK and the install reverted to the pinned 7.13 backend. Not yet re-checked with the 2026.44.1 pin, which fetches TheRock elsewhere; delete this row if the Windows nightly lane reports it as a stale XPASS." [["serve-large-model-inference"]] when = { effective_engine = "lemonade", os = "windows" } bug = "ROCMAI-900" -reason = "The Windows large-model serve crashes in the SDK's Tensile library on the first inference request; the pinned ROCm 7.13 backend runs against the 7.14.1 SDK because Lemonade 11.5.1 cannot align it (suspected cause, not reproduced off Windows)." +reason = "The Windows large-model serve crashed in the SDK's Tensile library on the first inference request while Lemonade 11.5.1 was pinned: its ROCm 7.13 backend ran against the 7.14.1 SDK because 11.5.1 could not align it (suspected cause, not reproduced off Windows). Not yet re-checked with the 2026.44.1 pin; delete this row if the Windows nightly lane reports it as a stale XPASS." From 0b2add5b0ac1be428e38ce703cdf842d5014df05 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 13 Nov 2026 11:34:29 +0200 Subject: [PATCH 06/13] test(e2e): check the llama.cpp ROCm tag where the SDK is the packaged pin On a ROCm 10.1 lane the active SDK equals the pin Lemonade 2026.44.1 ships, so comparing the ROCm version cannot show a rewritten pin. Also compare the paired llamacpp.rocm-stable tag, and name the e2e-test-hooks feature in the manual-testing note. Signed-off-by: Roman Inflianskas --- docs/manual-testing.md | 2 +- tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 46 ++++++++++++++----- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 8e98d8c570..870177fbb3 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -396,7 +396,7 @@ server (no download needed): the pinned build needs glibc 2.38 and OpenSSL 3 (`libssl.so.3`). On a real older host, such as Ubuntu 22.04 (glibc 2.35) or Ubuntu 20.04 (glibc 2.31, OpenSSL 1.1 only), run `rocm engines install lemonade --yes` in a fresh data folder. On a newer host, -use a debug build and set `ROCM_E2E_LEMONADE_HOST_GLIBC=2.31` and +use a build with the `rocm/e2e-test-hooks` feature and set `ROCM_E2E_LEMONADE_HOST_GLIBC=2.31` and `ROCM_E2E_LEMONADE_HOST_OPENSSL=missing` first. Expected result: - It fails with `this host's libraries are too old for the Lemonade diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index 2fcfaf980b..d9f9ea0388 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -565,6 +565,17 @@ async fn assert_lemonade_backend_paired_with_active_sdk(world: &mut E2eWorld) { pinned_version, active_version, "the installed backend_versions.json does not pin the active ROCm SDK:\nstdout:\n{output}\nstderr:\n{stderr}" ); + if pinned_version == packaged_therock_pin(output) { + // The active SDK is the packaged pin (ROCm 10.1 against Lemonade + // 2026.44.1), so the comparison above holds whatever the CLI did. The + // llama.cpp tag paired with that pin is a second signal: an alignment + // that rewrote the pin would also have moved it. + assert_eq!( + installed_backend_pin(output, "llamacpp", "rocm-stable"), + packaged_backend_pin(output, "llamacpp", "rocm-stable"), + "the active SDK is the packaged pin, yet the llama.cpp ROCm tag was rewritten:\nstdout:\n{output}\nstderr:\n{stderr}" + ); + } let reported_aligned = output .contains("Aligned Lemonade's ROCm llama.cpp backend to match the installed ROCm SDK"); if !reported_aligned { @@ -592,6 +603,13 @@ fn is_plain_release_version(version: &str) -> bool { /// `therock.version` in the `resources/backend_versions.json` of the runtime /// tree the install reports (`env_path:`). fn installed_therock_pin(output: &str) -> String { + installed_backend_pin(output, "therock", "version") + .unwrap_or_else(|| panic!("no therock.version in the installed backend_versions.json")) +} + +/// `
.` in the installed `resources/backend_versions.json`, e.g. +/// `llamacpp.rocm-stable`, the llama.cpp tag that goes with the ROCm pin. +fn installed_backend_pin(output: &str, section: &str, key: &str) -> Option { let env_path = output .lines() .find_map(|line| line.trim().strip_prefix("env_path: ")) @@ -610,10 +628,7 @@ fn installed_therock_pin(output: &str) -> String { backend_versions_path.display() ) }); - parsed["therock"]["version"] - .as_str() - .unwrap_or_else(|| panic!("no therock.version in {}", backend_versions_path.display())) - .to_owned() + parsed[section][key].as_str().map(str::to_owned) } /// The CLI names the variable whenever it is set, so a user can tell it was @@ -695,13 +710,18 @@ async fn assert_packaged_pin_survives(world: &mut E2eWorld) { let pinned_version = installed_therock_pin(output); let packaged_version = packaged_therock_pin(output); // Where the active SDK already is the packaged pin (a ROCm 10.1 SDK against - // Lemonade 2026.44.1) this cannot tell an untouched pin from one rewritten to - // the same value; the absent alignment and revert lines above are then the - // whole check. + // Lemonade 2026.44.1) the ROCm version alone cannot tell an untouched pin + // from one rewritten to the same value, so the llama.cpp tag paired with it + // is checked too. assert_eq!( pinned_version, packaged_version, "the pin differs from the one the Lemonade archive ships even though the user opted out" ); + assert_eq!( + installed_backend_pin(output, "llamacpp", "rocm-stable"), + packaged_backend_pin(output, "llamacpp", "rocm-stable"), + "the llama.cpp ROCm tag differs from the one the Lemonade archive ships even though the user opted out" + ); } /// `therock.version` as the Lemonade embeddable archive the install verified @@ -712,6 +732,13 @@ async fn assert_packaged_pin_survives(world: &mut E2eWorld) { /// (`lemonade-embeddable-`): the downloads directory is never pruned, /// so a runner that installed an earlier pin still holds that archive too. fn packaged_therock_pin(output: &str) -> String { + packaged_backend_pin(output, "therock", "version") + .unwrap_or_else(|| panic!("no therock.version in the packaged backend_versions.json")) +} + +/// `
.` as the cached Lemonade archive ships it; see +/// [`packaged_therock_pin`]. +fn packaged_backend_pin(output: &str, section: &str, key: &str) -> Option { let env_path = output .lines() .find_map(|line| line.trim().strip_prefix("env_path: ")) @@ -759,10 +786,7 @@ fn packaged_therock_pin(output: &str) -> String { ); let parsed: serde_json::Value = serde_json::from_slice(&extracted.stdout) .unwrap_or_else(|error| panic!("failed to parse {member}: {error}")); - parsed["therock"]["version"] - .as_str() - .unwrap_or_else(|| panic!("no therock.version in {member}")) - .to_owned() + parsed[section][key].as_str().map(str::to_owned) } /// The engine inventory reports a usable engine runtime. From 96956bfb9543b8f032fd461e4f85cf83ff552cf8 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 16 Nov 2026 12:59:10 +0200 Subject: [PATCH 07/13] style(lemonade): cargo fmt the spawn helpers Signed-off-by: Roman Inflianskas --- engines/lemonade/src/process.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs index 7e325679e9..80e42a494b 100644 --- a/engines/lemonade/src/process.rs +++ b/engines/lemonade/src/process.rs @@ -100,7 +100,15 @@ pub(crate) fn spawn_lemond_with_parent( log_mode, ); #[cfg(not(target_os = "linux"))] - spawn_lemond_unchecked(manifest, host, port, log_path, process_env, parent, log_mode) + spawn_lemond_unchecked( + manifest, + host, + port, + log_path, + process_env, + parent, + log_mode, + ) } /// [`spawn_lemond_with_parent`] on Linux, with the host's libraries given so a @@ -118,7 +126,15 @@ fn spawn_lemond_on( log_mode: LogMode, ) -> Result { refuse_lemond_on_too_old_host(manifest, host_libraries)?; - spawn_lemond_unchecked(manifest, host, port, log_path, process_env, parent, log_mode) + spawn_lemond_unchecked( + manifest, + host, + port, + log_path, + process_env, + parent, + log_mode, + ) } /// Refuse to start the pinned Linux `lemond` on a host whose glibc or OpenSSL From c03c7b0b7ace414a6ca7e587a76023fc044fbe00 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 16 Nov 2026 17:34:16 +0200 Subject: [PATCH 08/13] test(e2e): stop calling Lemonade 11.5.1 the pin in a row-scope test comment Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/src/expectation.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 9ebdca6c6b..22f80c157f 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -2356,7 +2356,8 @@ flaky = true )); } - /// The two rows for Lemonade 11.5.1's failed backend alignment apply to native + /// The two rows for the Windows Lemonade backend problems (written while 11.5.1 + /// was pinned, kept until a nightly shows them stale) apply to native /// Windows with lemonade and to nothing else: not to Linux hosts (including one whose /// default engine is vLLM, where the engine named is what decides), and they are not /// `flaky` (an XPASS means the cause is gone and the row is stale). Neither sets From 9764b29c58ab6e07bf56db5a2e06aab83a38cede Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 12:31:44 +0200 Subject: [PATCH 09/13] fix(lemonade): add an opt-out for the host-library check, and say what a refusal left undone The check reads only the ldconfig cache, so a host that provides OpenSSL 3 or glibc some other way (LD_LIBRARY_PATH, a Nix or Guix profile) was refused with no way past it. ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK turns both refusals into a warning, and the refusal names it (lemonade-installs-08). A refused re-run where the pinned Lemonade is already installed said "Nothing was downloaded." about a download it would never have made; it now says the installed copy stays as it is (lemonade-installs-09). Also: the backends parser no longer ends a recipe at a first-column line with no state; runtime-16 requires the install to report an alignment whenever the pin moved; and the ELF reader test skips on a host without ldconfig. Signed-off-by: Roman Inflianskas --- README.md | 7 +- docs/engine-plugins.md | 9 +- docs/manual-testing.md | 12 +- docs/testing.md | 11 ++ engines/lemonade/src/footprint.rs | 71 +++++++++ engines/lemonade/src/host_libraries.rs | 145 ++++++++++++++++-- engines/lemonade/src/lib.rs | 8 +- engines/lemonade/src/llamacpp_backend.rs | 52 ++++++- engines/lemonade/src/process.rs | 38 ++++- .../features/lemonade_installs.feature | 30 +++- .../features/runtime_setup.feature | 3 +- tests/e2e-cucumber/tests/e2e.rs | 4 + .../tests/e2e/lemonade_install_steps.rs | 129 ++++++++++++++++ tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 27 ++-- 14 files changed, 502 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index f4e725eada..2cafd63eeb 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,12 @@ Ubuntu 22.04 ships glibc 2.35 and cannot run it; Ubuntu 24.04 provides glibc 2.39, `GLIBCXX_3.4.33` and OpenSSL 3.0.13. On an older host, `rocm engines install lemonade` stops before downloading anything and says what falls short: the glibc version Lemonade needs and the one the host has, or that -the host has no `libssl.so.3`, the library of OpenSSL 3. +the host has no `libssl.so.3`, the library of OpenSSL 3. Where Lemonade is +already installed, the same check refuses to reinstall it or start it, and says +the installed copy was left as it is. The check reads the `ldconfig` cache, so a +host that provides these libraries some other way (`LD_LIBRARY_PATH`, a Nix or +Guix profile) can set `ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK=1`: the +shortfall is then printed as a warning and Lemonade goes ahead. > [!IMPORTANT] > **Tech Preview:** This software is provided as-is, without warranty or diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index 1f6b377644..eae236c131 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -155,7 +155,14 @@ library that falls short, the message names the version needed and what the host has: its version, read from the library the `ldconfig` cache lists, or that the cache has no such library at all. A host with only OpenSSL 1.1 has no `libssl.so.3`, so it reads that way. A host whose cache cannot be read, or -whose library does not parse, is not refused. Lemonade +whose library does not parse, is not refused. A refused install says what it +left undone: "Nothing was downloaded." for a fresh install, or that the +Lemonade already installed there stays as it is. Because only the `ldconfig` +cache is read, a host that provides OpenSSL 3 or glibc outside it +(`LD_LIBRARY_PATH`, a Nix or Guix profile, a user prefix, a stale cache) is +refused although `lemond` could load there; setting +`ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK` (any value) turns both refusals +into a warning, and the refusal message names it. Lemonade 2026.44.1, the current pin, needs glibc 2.38 and OpenSSL 3.0.0, which Ubuntu 24.04 meets; 2026.45.3's `lemond` needs glibc 2.43 and OpenSSL 3.2.0, which it does not. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index f7bd4c6e66..bb47a292cb 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -480,9 +480,19 @@ use a build with the `rocm/e2e-test-hooks` feature and set `ROCM_E2E_LEMONADE_HO - It fails with `this host's libraries are too old for the Lemonade server: it needs glibc 2.38 or newer (this host has glibc )`, followed on a host without OpenSSL 3 by `and OpenSSL 3.0.0 or newer (this host's loader - cache has no libssl.so.3)`, and ends `Nothing was downloaded.`. + cache has no libssl.so.3)`, then `Nothing was downloaded.`, and ends by + naming `ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK=1` as the way to skip + the check. - The data folder's `engines/lemonade` has no `downloads` or `runtime` subfolder, and `rocm storage report` lists no Lemonade install. +- Run it again with `ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK=1`: it prints + the same shortfall as a `warning:` ending `going ahead because + ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK is set.` and goes on to install + (on a real older host the server then fails to start). +- Where the pinned Lemonade is already installed, the refusal ends `Nothing was + changed: the Lemonade already installed in stays as it is, + but its server cannot start on this host either.` instead, and that folder is + unchanged. - On Ubuntu 24.04 or newer the same command installs Lemonade as usual. With room to spare, on Windows from an interactive console: install Lemonade diff --git a/docs/testing.md b/docs/testing.md index 2781b5c252..781a56ace8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1959,6 +1959,17 @@ Lemonade copies ROCm CLI keeps (ROCMAI-825), all on the mock lane: start-time refusal: `rocm serve` runs the install check first, a Linux serve with a packaged `llama-server` never starts `lemond`, and the mock lane has no GPU for serve to resolve. +- `lemonade-installs-08` (Linux) sets the same old-host hooks plus + `ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK=1`, and reports the drive + nearly full. It checks that the shortfall is printed as a warning naming the + variable, and that the install went on past the check to the space refusal + without downloading anything. The opt-out on the `lemond` start is + unit-tested in [process.rs](../engines/lemonade/src/process.rs). +- `lemonade-installs-09` (Linux) plants a tree of the pinned Lemonade in the + data folder, reads the pinned version from a first refused run, and re-runs + the install on the old-host hooks. It checks that the refusal says the + installed copy stays as it is rather than "Nothing was downloaded.", and that + every file of the planted tree is unchanged. - `lemonade-installs-03` plants a data-folder copy and a copy inside a runtime and checks `rocm storage report` lists both and flags the unused one. - `lemonade-installs-04` uninstalls that runtime and checks the install record diff --git a/engines/lemonade/src/footprint.rs b/engines/lemonade/src/footprint.rs index 9acdf26ccb..ecca264d8c 100644 --- a/engines/lemonade/src/footprint.rs +++ b/engines/lemonade/src/footprint.rs @@ -135,6 +135,35 @@ pub(crate) fn plan_install_preflight( }) } +/// The runtime folder of a tree of the pinned Lemonade already extracted where +/// this install goes, or `None` when the install would download and extract one. +pub(crate) fn pinned_tree_in_place(paths: &AppPaths, env_root: Option<&Path>) -> Option { + let runtime_dir = runtime_dir_in(&lemonade_root(paths, env_root)); + let extraction = needs_extraction( + false, + installed_runtime_version(&runtime_dir).as_deref(), + rocm_deps::LEMONADE_VERSION, + lemond_path_in(&runtime_dir).is_file(), + ); + (!extraction).then_some(runtime_dir) +} + +/// What a host-library refusal of an install left undone, for the branch the +/// install was on: a fresh install had a download ahead of it, while a tree of +/// the pinned version already in place had nothing to fetch and is left as it +/// is, though it cannot start either. +pub(crate) fn host_refusal_outcome(installed_tree: Option<&Path>) -> String { + match installed_tree { + None => "Nothing was downloaded.".to_owned(), + Some(tree) => format!( + "Nothing was changed: the Lemonade {} already installed in {} stays as it is, \ + but its server cannot start on this host either.", + rocm_deps::LEMONADE_VERSION, + tree.display() + ), + } +} + /// Total size of the files under `dir`, best effort: symlinks are not followed /// and anything unreadable counts as zero, because this only sizes the space an /// install is about to free. @@ -491,6 +520,48 @@ mod tests { const GIB: u64 = 1024 * 1024 * 1024; + /// The host-library refusal's last line follows the install's branch: no + /// tree, or a tree of another version, would have been downloaded, so + /// "Nothing was downloaded."; a tree of the pinned version would not, so + /// the line says it was left in place instead. + #[test] + fn the_host_refusal_outcome_follows_whether_the_pinned_tree_is_in_place() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let base = install_tree_root(&paths, None); + + let fresh = pinned_tree_in_place(&paths, None); + assert_eq!(fresh, None, "an empty folder has no tree in place"); + assert_eq!( + host_refusal_outcome(fresh.as_deref()), + "Nothing was downloaded." + ); + + plant_current_tree(&base); + let runtime_dir = runtime_dir_in(&base); + crate::install::record_runtime_version(&runtime_dir, "0.0.1").unwrap(); + assert_eq!( + pinned_tree_in_place(&paths, None), + None, + "a tree of another version is replaced, so it counts as not in place" + ); + + crate::install::record_runtime_version(&runtime_dir, rocm_deps::LEMONADE_VERSION).unwrap(); + let in_place = pinned_tree_in_place(&paths, None); + assert_eq!(in_place.as_deref(), Some(runtime_dir.as_path())); + let outcome = host_refusal_outcome(in_place.as_deref()); + assert_eq!( + outcome, + format!( + "Nothing was changed: the Lemonade {} already installed in {} stays as it \ + is, but its server cannot start on this host either.", + rocm_deps::LEMONADE_VERSION, + runtime_dir.display() + ) + ); + assert!(!outcome.contains("downloaded"), "{outcome}"); + } + /// The release-test sequence: Lemonade installed into the data folder, then /// an SDK runtime activated, so the next install targets the runtime's own /// folder. The notice must name the new folder and the copy it is not reusing. diff --git a/engines/lemonade/src/host_libraries.rs b/engines/lemonade/src/host_libraries.rs index d07ed307c6..3043f605cc 100644 --- a/engines/lemonade/src/host_libraries.rs +++ b/engines/lemonade/src/host_libraries.rs @@ -167,18 +167,74 @@ pub(crate) fn host_shortfall( )) } -/// Fail with [`host_shortfall`]'s message plus `outcome`, which says what the -/// refusal left undone ("Nothing was downloaded."). +/// Set (to any value, empty included) to start the pinned `lemond` even where +/// [`host_shortfall`] says the host falls short: the shortfall is then printed +/// as a warning instead of refusing. For a host that provides glibc or OpenSSL 3 +/// outside the loader cache the probe reads (`LD_LIBRARY_PATH`, a Nix or Guix +/// profile, a user prefix, a stale cache). Presence is the signal, as for +/// [`crate::backend_alignment::LEMONADE_BACKEND_ALIGNMENT_DISABLED_ENV`]. +pub const LEMONADE_HOST_LIBRARY_CHECK_DISABLED_ENV: &str = + "ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK"; + +/// Whether the user has turned the host-library check off. +#[cfg(target_os = "linux")] +pub(crate) fn host_library_check_disabled() -> bool { + std::env::var_os(LEMONADE_HOST_LIBRARY_CHECK_DISABLED_ENV).is_some() +} + +/// What the host-library check decided. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum HostCheck { + /// Nothing short of the needs was found. + Pass, + /// The host falls short, but the check is turned off: go ahead and say so. + Warn(String), + /// The host falls short: stop, with this message. + Refuse(String), +} + +/// Decide from [`host_shortfall`]: refuse with its message, then `outcome` +/// (what the refusal left undone, such as "Nothing was downloaded."), then how +/// to turn the check off; or, when `check_disabled`, warn instead. +pub(crate) fn host_check( + lemonade_version: &str, + needs: &LemondHostNeeds, + host: &HostLibraries, + outcome: &str, + check_disabled: bool, +) -> HostCheck { + let Some(shortfall) = host_shortfall(lemonade_version, needs, host) else { + return HostCheck::Pass; + }; + if check_disabled { + return HostCheck::Warn(format!( + "warning: {shortfall}; going ahead because \ + {LEMONADE_HOST_LIBRARY_CHECK_DISABLED_ENV} is set." + )); + } + HostCheck::Refuse(format!( + "{shortfall}. {outcome} If this host provides these libraries outside the \ + loader cache (LD_LIBRARY_PATH, a Nix or Guix profile), set \ + {LEMONADE_HOST_LIBRARY_CHECK_DISABLED_ENV}=1 to skip this check." + )) +} + +/// Apply [`host_check`]: fail on a refusal, print a warning to stderr. pub(crate) fn refuse_if_host_too_old( lemonade_version: &str, needs: &LemondHostNeeds, host: &HostLibraries, outcome: &str, + check_disabled: bool, ) -> Result<()> { - if let Some(shortfall) = host_shortfall(lemonade_version, needs, host) { - bail!("{shortfall}. {outcome}"); + match host_check(lemonade_version, needs, host, outcome, check_disabled) { + HostCheck::Pass => Ok(()), + HostCheck::Warn(warning) => { + eprintln!("{warning}"); + Ok(()) + } + HostCheck::Refuse(message) => bail!(message), } - Ok(()) } /// The newest numeric version among `names` that start with `prefix` @@ -403,20 +459,76 @@ mod tests { } #[test] - fn refusal_appends_what_it_left_undone_and_passes_a_new_enough_host() { - let error = refuse_if_host_too_old( + fn refusal_appends_what_it_left_undone_and_how_to_skip_the_check() { + let HostCheck::Refuse(message) = host_check( "9.9.9", &NEEDS, &host(Some("2.35"), None), "Nothing was downloaded.", - ) - .unwrap_err() - .to_string(); + false, + ) else { + panic!("a host short of glibc was not refused"); + }; assert!( - error.ends_with("(this host has glibc 2.35). Nothing was downloaded."), - "{error}" + message.contains("(this host has glibc 2.35). Nothing was downloaded. If this host"), + "{message}" + ); + assert!( + message.ends_with( + "set ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK=1 to skip this check." + ), + "{message}" + ); + assert!( + refuse_if_host_too_old("9.9.9", &NEEDS, &host(Some("2.35"), None), "x", false).is_err() + ); + assert_eq!( + host_check( + "9.9.9", + &NEEDS, + &host(Some("2.39"), Some("3.0.13")), + "x", + false + ), + HostCheck::Pass + ); + refuse_if_host_too_old( + "9.9.9", + &NEEDS, + &host(Some("2.39"), Some("3.0.13")), + "x", + false, + ) + .unwrap(); + } + + /// With the check turned off, a host that falls short is let through, and + /// the warning still names what it falls short of and why it went ahead. + #[test] + fn a_disabled_check_warns_instead_of_refusing() { + let missing_ssl = host(Some("2.31"), Some("missing")); + let HostCheck::Warn(warning) = host_check("9.9.9", &NEEDS, &missing_ssl, "x", true) else { + panic!("a disabled check did not warn"); + }; + assert_eq!( + warning, + "warning: this host's libraries are too old for the Lemonade 9.9.9 server: it \ + needs glibc 2.38 or newer (this host has glibc 2.31) and OpenSSL 3.0.0 or newer \ + (this host's loader cache has no libssl.so.3); going ahead because \ + ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK is set." + ); + refuse_if_host_too_old("9.9.9", &NEEDS, &missing_ssl, "x", true).unwrap(); + // A host that meets the needs prints nothing either way. + assert_eq!( + host_check( + "9.9.9", + &NEEDS, + &host(Some("2.39"), Some("3.0.13")), + "x", + true + ), + HostCheck::Pass ); - refuse_if_host_too_old("9.9.9", &NEEDS, &host(Some("2.39"), Some("3.0.13")), "x").unwrap(); } const LIBC_LINE: &str = @@ -515,7 +627,12 @@ mod tests { #[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))] #[test] fn the_running_hosts_glibc_defines_a_readable_version() { - let cache = rocm_core::ldconfig_cache().expect("ldconfig -p"); + // A host without `ldconfig` (a minimal container, NixOS) has no cache + // to read; the probe reads that as unknown, which other tests cover. + let Some(cache) = rocm_core::ldconfig_cache() else { + eprintln!("skipped: `ldconfig -p` could not be run on this host"); + return; + }; let path = x86_64_library_path(&cache, "libc.so.6").expect("libc.so.6 listed"); let found = library_version(path, "GLIBC_").expect("GLIBC_ version nodes"); assert!(found >= LibVersion::parse("2.17").unwrap(), "{found}"); diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 9896d65517..5140adfbd1 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -418,12 +418,18 @@ fn install_response(request: InstallRequest) -> Result { // Refused before anything is downloaded: a `lemond` built against a newer // glibc or OpenSSL than the host has exits at once with a loader error, // which used to reach the user only as "did not become ready". + // The line saying what the refusal left undone follows the branch: an + // install whose pinned tree is already in place would have downloaded + // nothing anyway. #[cfg(target_os = "linux")] crate::host_libraries::refuse_if_host_too_old( rocm_deps::LEMONADE_VERSION, &crate::install::LINUX_LEMOND_NEEDS, &crate::host_libraries::probe_host_libraries(), - "Nothing was downloaded.", + &crate::footprint::host_refusal_outcome( + crate::footprint::pinned_tree_in_place(&paths, env_root.as_deref()).as_deref(), + ), + crate::host_libraries::host_library_check_disabled(), )?; // Announced and space-checked before anything is downloaded: a fresh tree // is a multi-gigabyte install, and `rocm serve` reaches this on its own. diff --git a/engines/lemonade/src/llamacpp_backend.rs b/engines/lemonade/src/llamacpp_backend.rs index 761f91047b..54efdef375 100644 --- a/engines/lemonade/src/llamacpp_backend.rs +++ b/engines/lemonade/src/llamacpp_backend.rs @@ -172,11 +172,18 @@ pub(crate) fn ensure_best_llamacpp_backend_with_parent( /// release out) as `installed`, and `update_required` (an installed build /// Lemonade will not run until it is reinstalled, e.g. after the pin in /// `backend_versions.json` moved) as `installable`, so it is reinstalled rather -/// than reused or dropped. The other state Lemonade defines, `action_required` +/// than reused or dropped. Lemonade 2026.44.1's own `Action` for such a row is +/// `lemonade backends install :`, the command rocm-cli runs, +/// and its `BackendManager::install_backend` has no early return for an +/// installed backend while online; for `llamacpp:rocm` it also removes and +/// fetches TheRock again (`src/cpp/server/system_info.cpp` and +/// `src/cpp/server/backend_manager.cpp` at that tag). The other state Lemonade defines, `action_required` /// (used by other recipes), leaves the row out. /// -/// A row that starts in the first column names its recipe; an indented row -/// continues the recipe above it with another backend. The recipe is read from +/// A row that starts in the first column and carries a state names its recipe; +/// an indented row continues the recipe above it with another backend. A +/// first-column line with no state, such as a warning, is skipped and leaves +/// the current recipe as it was. The recipe is read from /// the layout rather than matched against a list of known recipe names: each /// Lemonade release adds recipes (`llamacpp-hrx`, `ds4`, `thenoise` in /// 2026.44), and a row for a recipe missing from such a list used to be read as @@ -217,11 +224,18 @@ fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { let (backend, rest) = if continues_recipe { (first, &tokens[1..]) } else { - first.clone_into(&mut current_recipe); - match tokens.get(1) { - Some(backend) => (*backend, &tokens[2..]), - None => continue, + let (Some(backend), Some(rest)) = (tokens.get(1), tokens.get(2..)) else { + continue; + }; + // Only a row with a state in it names a recipe. Any other + // first-column line (a warning Lemonade printed mid-table) is + // skipped without ending the recipe above it, so the indented + // rows that follow still count for that recipe. + if !rest.iter().any(|token| is_state(token)) { + continue; } + first.clone_into(&mut current_recipe); + (*backend, rest) }; if current_recipe != LLAMACPP_RECIPE { continue; @@ -491,6 +505,30 @@ thinksound rocm experimental native installable Backend ); } + /// A first-column line with no state between a `llamacpp` row and its + /// indented rows is not a recipe: the rows after it are still `llamacpp`'s. + #[test] + fn a_warning_line_inside_the_table_does_not_end_the_llamacpp_rows() { + let table = "\ +Recipe Backend Tier Format Status Message/Version Action +--------------------------------------------------------------------------------------------------------------- +llamacpp cpu core native installable Backend is supported but not installed. lemonade backends install llamacpp:cpu +Warning: could not reach GitHub to check for updates + rocm core native update_required Backend update is required before use. lemonade backends install llamacpp:rocm + vulkan core native installed b11580 +sd-cpp cpu core native installable Backend is supported but not installed. lemonade backends install sd-cpp:cpu + rocm core native installable Backend is supported but not installed. lemonade backends install sd-cpp:rocm +"; + assert_eq!( + parse_llamacpp_backend_statuses(table), + vec![ + ("cpu".to_owned(), "installable".to_owned()), + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "installed".to_owned()), + ] + ); + } + #[test] fn another_recipes_backend_is_never_read_as_a_llamacpp_backend() { // Plain `lemonade backends` (no `--all`) leaves unsupported backends diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs index 80e42a494b..c07507f344 100644 --- a/engines/lemonade/src/process.rs +++ b/engines/lemonade/src/process.rs @@ -91,6 +91,7 @@ pub(crate) fn spawn_lemond_with_parent( #[cfg(target_os = "linux")] return spawn_lemond_on( &crate::host_libraries::probe_host_libraries(), + crate::host_libraries::host_library_check_disabled(), manifest, host, port, @@ -111,12 +112,14 @@ pub(crate) fn spawn_lemond_with_parent( ) } -/// [`spawn_lemond_with_parent`] on Linux, with the host's libraries given so a -/// test can drive the refusal on a host whose libraries are new. +/// [`spawn_lemond_with_parent`] on Linux, with the host's libraries and the +/// opt-out given so a test can drive the refusal on a host whose libraries are +/// new. #[cfg(target_os = "linux")] #[allow(clippy::too_many_arguments)] fn spawn_lemond_on( host_libraries: &crate::host_libraries::HostLibraries, + check_disabled: bool, manifest: &LemonadeInstallManifest, host: &str, port: u16, @@ -125,7 +128,7 @@ fn spawn_lemond_on( parent: &ParentRuntimeEnvironment, log_mode: LogMode, ) -> Result { - refuse_lemond_on_too_old_host(manifest, host_libraries)?; + refuse_lemond_on_too_old_host(manifest, host_libraries, check_disabled)?; spawn_lemond_unchecked( manifest, host, @@ -148,6 +151,7 @@ fn spawn_lemond_on( fn refuse_lemond_on_too_old_host( manifest: &LemonadeInstallManifest, host_libraries: &crate::host_libraries::HostLibraries, + check_disabled: bool, ) -> Result<()> { if manifest.version != rocm_deps::LEMONADE_VERSION { return Ok(()); @@ -157,6 +161,7 @@ fn refuse_lemond_on_too_old_host( &crate::install::LINUX_LEMOND_NEEDS, host_libraries, "Lemonade was not started.", + check_disabled, ) } @@ -941,12 +946,14 @@ mod tests { fn start_on( manifest: &LemonadeInstallManifest, host_libraries: &crate::host_libraries::HostLibraries, + check_disabled: bool, ) -> Result<()> { let mut attempts = 0; let parent_root = tempfile::tempdir().unwrap(); let mut child = loop { match spawn_lemond_on( host_libraries, + check_disabled, manifest, "127.0.0.1", 1, @@ -997,7 +1004,7 @@ mod tests { #[test] fn the_pinned_lemond_is_not_started_on_a_host_with_too_old_libraries() { let (_dir, manifest, marker) = marker_lemond(rocm_deps::LEMONADE_VERSION); - let error = start_on(&manifest, &libraries("2.31", "missing")) + let error = start_on(&manifest, &libraries("2.31", "missing"), false) .expect_err("a host older than the pinned lemond needs is refused") .to_string(); let needs = &crate::install::LINUX_LEMOND_NEEDS; @@ -1011,7 +1018,10 @@ mod tests { ] { assert!(error.contains(&expected), "missing `{expected}`: {error}"); } - assert!(error.ends_with("Lemonade was not started."), "{error}"); + assert!( + error.contains("Lemonade was not started. If this host provides"), + "{error}" + ); assert!(!marker.exists(), "lemond ran despite the refusal"); } @@ -1022,20 +1032,34 @@ mod tests { fn lemond_starts_on_a_new_enough_host_and_for_a_tree_of_another_version() { let needs = &crate::install::LINUX_LEMOND_NEEDS; let (_dir, manifest, marker) = marker_lemond(rocm_deps::LEMONADE_VERSION); - start_on(&manifest, &libraries(needs.glibc, needs.openssl)).unwrap(); + start_on(&manifest, &libraries(needs.glibc, needs.openssl), false).unwrap(); assert!( marker.exists(), "lemond did not start on a host meeting its needs" ); let (_dir, manifest, marker) = marker_lemond("0.0.1"); - start_on(&manifest, &libraries("2.31", "missing")).unwrap(); + start_on(&manifest, &libraries("2.31", "missing"), false).unwrap(); assert!( marker.exists(), "a tree of another version was refused by this pin's needs" ); } + /// With `ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK` set, the pinned + /// `lemond` starts on a host the check would refuse, as the refusal's own + /// advice promises. + #[cfg(target_os = "linux")] + #[test] + fn a_disabled_check_starts_the_pinned_lemond_on_a_host_it_would_refuse() { + let (_dir, manifest, marker) = marker_lemond(rocm_deps::LEMONADE_VERSION); + start_on(&manifest, &libraries("2.31", "missing"), true).unwrap(); + assert!( + marker.exists(), + "the disabled check still kept lemond from starting" + ); + } + /// Build the real backend-install command for a console state and report /// whether the window was hidden, along with the command itself. fn backend_install_for(consoles: OutputConsoles) -> (ProcessCommand, bool) { diff --git a/tests/e2e-cucumber/features/lemonade_installs.feature b/tests/e2e-cucumber/features/lemonade_installs.feature index aba44b175f..8b79591a37 100644 --- a/tests/e2e-cucumber/features/lemonade_installs.feature +++ b/tests/e2e-cucumber/features/lemonade_installs.feature @@ -16,8 +16,8 @@ Feature: Lemonade installs on disk # None downloads anything or needs a GPU, so all run on the mock lane every PR. # lemonade-installs-01 and -02 need the `rocm/e2e-test-hooks` build # `cargo xtask e2e` makes, to report a nearly full drive without filling a - # real one; lemonade-installs-07 needs it to report old host libraries on a - # host whose libraries are new. + # real one; lemonade-installs-07, -08 and -09 need it to report old host + # libraries on a host whose libraries are new. @id:lemonade-installs-refused-without-space Scenario: lemonade-installs-01 - A Lemonade install the drive cannot hold is refused before downloading @@ -81,3 +81,29 @@ Feature: Lemonade installs on disk Given this host's glibc is older than the pinned Lemonade server needs and it has no OpenSSL 3 When the user installs the Lemonade engine Then the Lemonade install is refused for the host's libraries before anything is downloaded + + # A host can provide OpenSSL 3 or glibc outside the loader cache the check + # reads (LD_LIBRARY_PATH, a Nix or Guix profile, a user prefix), and the + # refusal names an opt-out for it. The opt-out must actually get the install + # past the check: here it goes on until the drive, reported nearly full, stops + # it before any download, which proves the host-library check let it through + # without fetching anything. + @id:lemonade-installs-host-library-check-opt-out @requires-os:linux + Scenario: lemonade-installs-08 - Turning the host-library check off lets a Lemonade install go ahead with a warning + Given this host's glibc is older than the pinned Lemonade server needs and it has no OpenSSL 3 + And the user has turned off the Lemonade host-library check + And the drive for a new Lemonade install cannot hold even Lemonade itself + When the user installs the Lemonade engine + Then the user is warned that the host's libraries are too old and the install went ahead + And the Lemonade install is refused before anything is downloaded + + # A re-run of the install where the pinned Lemonade is already in place has + # nothing to download, so "Nothing was downloaded." would describe a branch the + # install was never on. The refusal says the installed copy was left as it is + # instead, and it was. + @id:lemonade-installs-refused-on-too-old-host-libraries-when-installed @requires-os:linux + Scenario: lemonade-installs-09 - Re-running a Lemonade install on a host too old for it leaves the installed copy as it is + Given this host's glibc is older than the pinned Lemonade server needs and it has no OpenSSL 3 + And the pinned Lemonade is already installed in the data folder + When the user installs the Lemonade engine + Then the Lemonade install is refused for the host's libraries and the installed copy is left as it is diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index a6d90e5dd7..2114a7b355 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -342,7 +342,8 @@ Feature: Runtime configuration # packaged pin (ROCm 7.14 against 2026.44.1's 10.1.0) is aligned, and the CLI # says so; an SDK that already is the packaged pin (ROCm 10.1) needs no # alignment and gets none; a nightly SDK is never aligned and keeps the - # packaged pin. The Then accepts each route and fails a revert: + # packaged pin. The Then accepts each route and fails a revert, or an + # alignment the install does not report: # it reads the pin the installed runtime tree holds and compares it with the # active runtime's version. `--reinstall` re-extracts the packaged embeddable, # resetting `backend_versions.json` to its pinned defaults, so this does not diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 6e8db6c579..18c038c6df 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -240,6 +240,9 @@ pub struct E2eWorld { /// local webhook it listens on. Held here so `Drop` can kill and reap it /// whatever the scenario's outcome. pub automation_daemon: Option, + /// Every file of a Lemonade tree a Given planted, with its bytes, so a Then + /// can show a refused install left that tree exactly as it was. + pub planted_tree_snapshot: Option)>>, } /// One scenario's resolved expectation plus the identity needed to report it. @@ -409,6 +412,7 @@ impl Default for E2eWorld { recorded_descendant: None, dash_report_refused: None, automation_daemon: None, + planted_tree_snapshot: None, } } } diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs index 713c2facd9..5379c6e356 100644 --- a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -205,6 +205,135 @@ async fn assert_lemonade_install_refused_for_host_libraries(world: &mut E2eWorld ); } +#[given("the user has turned off the Lemonade host-library check")] +async fn host_library_check_turned_off(world: &mut E2eWorld) { + world + .command_env + .push(("ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK", "1".into())); +} + +/// The warning names the shortfall and the variable that let the install past +/// it; the next Then shows the install really went on (to the space check). +#[then("the user is warned that the host's libraries are too old and the install went ahead")] +async fn assert_host_library_warning(world: &mut E2eWorld) { + let output = combined_output(world); + assert!( + output.contains("warning: this host's libraries are too old for the Lemonade "), + "no host-library warning: the binary under test was probably built without \ + the `rocm/e2e-test-hooks` feature, which `cargo xtask e2e` enables:\n{output}" + ); + for expected in [ + format!("or newer (this host has glibc {OLD_HOST_GLIBC})"), + "going ahead because ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK is set.".to_owned(), + ] { + assert!( + output.contains(&expected), + "missing `{expected}`:\n{output}" + ); + } + assert!( + !output.contains("to skip this check."), + "the install was refused for the host's libraries despite the opt-out:\n{output}" + ); +} + +/// The Lemonade version this build pins, read black-box from the refusal the +/// old-host hooks produce on an empty data folder (lemonade-installs-07 shows +/// that run changes nothing): "... too old for the Lemonade server". +/// +/// A run consumes the scenario's environment, so the old-host hooks the Given +/// before this one set are put back for the When. +fn pinned_lemonade_version(world: &mut E2eWorld) -> String { + let scenario_env = world.command_env.clone(); + let (stdout, stderr, _) = + crate::run_rocm_with_scenario_env(world, &["engines", "install", "lemonade", "--yes"]); + world.command_env = scenario_env; + let output = format!("{stdout}\n{stderr}"); + output + .split("too old for the Lemonade ") + .nth(1) + .and_then(|rest| rest.split_whitespace().next()) + .unwrap_or_else(|| { + panic!( + "no host-library refusal to read the pinned version from; the binary under \ + test was probably built without the `rocm/e2e-test-hooks` feature:\n{output}" + ) + }) + .to_owned() +} + +/// What the planted tree holds, file by file, so the Then can show it was left +/// exactly as it was. +fn tree_files(root: &std::path::Path) -> Vec<(PathBuf, Vec)> { + let mut files = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir).expect("failed to list the planted tree") { + let path = entry.expect("failed to read a tree entry").path(); + if path.is_dir() { + stack.push(path); + } else { + let bytes = std::fs::read(&path).expect("failed to read a tree file"); + files.push((path, bytes)); + } + } + } + files.sort(); + files +} + +#[given("the pinned Lemonade is already installed in the data folder")] +async fn plant_pinned_lemonade_install(world: &mut E2eWorld) { + let version = pinned_lemonade_version(world); + let tree = data_folder_tree(world); + plant_lemonade_tree(&tree, DATA_FOLDER_TREE_BYTES); + // The marker `prepare_embeddable` writes after a complete extraction. + std::fs::write(tree.join("runtime").join("runtime-version.txt"), &version) + .expect("failed to write the runtime version marker"); + world.planted_tree_snapshot = Some(tree_files(&tree)); +} + +#[then( + "the Lemonade install is refused for the host's libraries and the installed copy is left as it is" +)] +async fn assert_refused_and_installed_copy_kept(world: &mut E2eWorld) { + let output = combined_output(world); + assert_ne!( + world.cli_rc, + Some(0), + "the install must fail on a host too old for the pinned Lemonade server:\n{output}" + ); + let tree = data_folder_tree(world); + let runtime_dir = tree.join("runtime"); + let expected = format!( + "already installed in {} stays as it is", + runtime_dir.display() + ); + for expected in [ + format!("or newer (this host has glibc {OLD_HOST_GLIBC})"), + "Nothing was changed: the Lemonade ".to_owned(), + expected, + ] { + assert!( + output.contains(&expected), + "missing `{expected}`:\n{output}" + ); + } + assert!( + !output.contains("Nothing was downloaded."), + "the refusal describes a download the re-run would never have made:\n{output}" + ); + // "Nothing was changed" is checked against the disk. + let before = world + .planted_tree_snapshot + .as_ref() + .expect("no planted tree snapshot - did the Given run?"); + assert!( + &tree_files(&tree) == before, + "the installed tree changed despite the refusal:\n{output}" + ); +} + #[then("the user is told what the Lemonade install needs and where it would go")] async fn assert_lemonade_install_notice(world: &mut E2eWorld) { let output = combined_output(world); diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index f1c73a3b2e..1912039f1e 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -691,7 +691,10 @@ async fn assert_lemonade_backend_paired_with_active_sdk(world: &mut E2eWorld) { pinned_version, active_version, "the installed backend_versions.json does not pin the active ROCm SDK:\nstdout:\n{output}\nstderr:\n{stderr}" ); - if pinned_version == packaged_therock_pin(output) { + let packaged_version = packaged_therock_pin(output); + let reported_aligned = output + .contains("Aligned Lemonade's ROCm llama.cpp backend to match the installed ROCm SDK"); + if pinned_version == packaged_version { // The active SDK is the packaged pin (ROCm 10.1 against Lemonade // 2026.44.1), so the comparison above holds whatever the CLI did. The // llama.cpp tag paired with that pin is a second signal: an alignment @@ -701,16 +704,22 @@ async fn assert_lemonade_backend_paired_with_active_sdk(world: &mut E2eWorld) { packaged_backend_pin(output, "llamacpp", "rocm-stable"), "the active SDK is the packaged pin, yet the llama.cpp ROCm tag was rewritten:\nstdout:\n{output}\nstderr:\n{stderr}" ); - } - let reported_aligned = output - .contains("Aligned Lemonade's ROCm llama.cpp backend to match the installed ROCm SDK"); - if !reported_aligned { - // Only right when the packaged pin already was the active SDK: then no - // alignment is attempted at all. + // No alignment is attempted at all then, so none may be reported or + // logged. assert!( - !stderr.contains("Aligned Lemonade's ROCm backend to") + !reported_aligned + && !stderr.contains("Aligned Lemonade's ROCm backend to") && !stderr.contains("could not align Lemonade's ROCm backend to"), - "an alignment ran but the install did not report it:\n{stderr}" + "the active SDK is the packaged pin, yet an alignment ran:\nstdout:\n{output}\nstderr:\n{stderr}" + ); + } else { + // The pin moved from the packaged {packaged_version} to the active SDK, + // so an alignment ran, and the install must say so: a CLI that aligns + // silently fails here. + assert!( + reported_aligned, + "the pin moved from the packaged {packaged_version} to {pinned_version}, but the \ + install did not report the alignment:\nstdout:\n{output}\nstderr:\n{stderr}" ); } } From 9605dbb6b447bafb5264271c73adcbf63b129677 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 13:21:32 +0200 Subject: [PATCH 10/13] fix(lemonade): compile the host-refusal outcome helpers on Linux only outside tests Their one caller, the host-library refusal, is Linux only, so a Windows build rejected them as dead code. Signed-off-by: Roman Inflianskas --- engines/lemonade/src/footprint.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/engines/lemonade/src/footprint.rs b/engines/lemonade/src/footprint.rs index ecca264d8c..e7608e274f 100644 --- a/engines/lemonade/src/footprint.rs +++ b/engines/lemonade/src/footprint.rs @@ -137,6 +137,10 @@ pub(crate) fn plan_install_preflight( /// The runtime folder of a tree of the pinned Lemonade already extracted where /// this install goes, or `None` when the install would download and extract one. +/// +/// Linux only outside tests, like its one caller, the host-library refusal +/// ([`crate::host_libraries`]): Windows has no such check. +#[cfg(any(target_os = "linux", test))] pub(crate) fn pinned_tree_in_place(paths: &AppPaths, env_root: Option<&Path>) -> Option { let runtime_dir = runtime_dir_in(&lemonade_root(paths, env_root)); let extraction = needs_extraction( @@ -152,6 +156,7 @@ pub(crate) fn pinned_tree_in_place(paths: &AppPaths, env_root: Option<&Path>) -> /// install was on: a fresh install had a download ahead of it, while a tree of /// the pinned version already in place had nothing to fetch and is left as it /// is, though it cannot start either. +#[cfg(any(target_os = "linux", test))] pub(crate) fn host_refusal_outcome(installed_tree: Option<&Path>) -> String { match installed_tree { None => "Nothing was downloaded.".to_owned(), From 3779afbacf7fe59e97662f3f15a5c48da14e2517 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 14:04:55 +0200 Subject: [PATCH 11/13] docs: stop saying every published Lemonade build needs glibc 2.38 2026.45.3 needs glibc 2.43, so the floor belongs to the pinned build. Reword it in docs/wsl.md, the fix-wsl-5 rationale `rocm fix` prints, the matching note `rocm diagnose` prints, and WSL_MIN_UBUNTU's doc comment. Also say that the host-library check does not read GLIBCXX, and that `object` is one new crate for the standalone Lemonade adapter binary. Signed-off-by: Roman Inflianskas --- README.md | 3 ++- crates/rocm-core/src/diagnose/wsl.rs | 2 +- crates/rocm-core/src/examine.rs | 4 ++-- crates/rocm-core/src/fix/catalog.rs | 2 +- docs/wsl.md | 6 +++--- engines/lemonade/Cargo.toml | 3 ++- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 6aadef3e96..15e9f4f493 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,8 @@ Ubuntu 22.04 ships glibc 2.35 and cannot run it; Ubuntu 24.04 provides glibc 2.39, `GLIBCXX_3.4.33` and OpenSSL 3.0.13. On an older host, `rocm engines install lemonade` stops before downloading anything and says what falls short: the glibc version Lemonade needs and the one the host has, or that -the host has no `libssl.so.3`, the library of OpenSSL 3. Where Lemonade is +the host has no `libssl.so.3`, the library of OpenSSL 3. That check reads glibc +and OpenSSL only; `GLIBCXX_3.4.32` (libstdc++) is not checked. Where Lemonade is already installed, the same check refuses to reinstall it or start it, and says the installed copy was left as it is. The check reads the `ldconfig` cache, so a host that provides these libraries some other way (`LD_LIBRARY_PATH`, a Nix or diff --git a/crates/rocm-core/src/diagnose/wsl.rs b/crates/rocm-core/src/diagnose/wsl.rs index d34fa9644f..3f601c28a8 100644 --- a/crates/rocm-core/src/diagnose/wsl.rs +++ b/crates/rocm-core/src/diagnose/wsl.rs @@ -266,7 +266,7 @@ pub(super) fn check_wsl_5_distro_too_old(e: &Examination, _symptom: &str) -> Dia verify: "f=/etc/os-release; [ -e \"$f\" ] || f=/usr/lib/os-release; grep VERSION_ID \"$f\" 2>/dev/null | head -1" .to_owned(), notes: vec![ - "This is a hard floor, not a recommendation: Ubuntu 22.04 ships glibc 2.35, below the glibc 2.38 / GLIBCXX_3.4.32 that every published Lemonade embeddable is linked against, so the engine cannot start there at all.".to_owned(), + "This is a hard floor, not a recommendation: Ubuntu 22.04 ships glibc 2.35, below the glibc 2.38 / GLIBCXX_3.4.32 that the pinned Lemonade embeddable is linked against (newer Lemonade releases need more), so the engine cannot start there at all.".to_owned(), ], ..Fix::default() }; diff --git a/crates/rocm-core/src/examine.rs b/crates/rocm-core/src/examine.rs index 98ad844b8d..269e81e864 100644 --- a/crates/rocm-core/src/examine.rs +++ b/crates/rocm-core/src/examine.rs @@ -86,8 +86,8 @@ pub struct Device { /// The oldest distro release the WSL path supports. /// /// Ubuntu 22.04 ships glibc 2.35, below the glibc 2.38 / `GLIBCXX_3.4.32` floor -/// every published Lemonade embeddable is linked against, so the engine cannot -/// start there. See `docs/wsl.md`. +/// the pinned Lemonade embeddable is linked against (newer Lemonade releases +/// need more), so the engine cannot start there. See `docs/wsl.md`. pub const WSL_MIN_UBUNTU: (u32, u32) = (24, 4); /// WSL2-specific machine state. `None` on every other platform. diff --git a/crates/rocm-core/src/fix/catalog.rs b/crates/rocm-core/src/fix/catalog.rs index 086a5e58a2..3affa51a61 100644 --- a/crates/rocm-core/src/fix/catalog.rs +++ b/crates/rocm-core/src/fix/catalog.rs @@ -732,7 +732,7 @@ pub(super) const RECIPES: &[FixRecipe] = &[ FixRecipe { fix_id: "fix-wsl-5-distro-too-old", title: "Move to a distro release the WSL path supports", - rationale: "Ubuntu 22.04 ships glibc 2.35, below the glibc 2.38 / GLIBCXX_3.4.32 floor every published Lemonade embeddable is linked against, so the engine cannot start there at all. This is a hard floor, not a recommendation.", + rationale: "Ubuntu 22.04 ships glibc 2.35, below the glibc 2.38 / GLIBCXX_3.4.32 floor the pinned Lemonade embeddable is linked against (newer Lemonade releases need more), so the engine cannot start there at all. This is a hard floor, not a recommendation.", commands: &[ "# From Windows, install a supported distro alongside the current one:", "# wsl --install -d Ubuntu-24.04", diff --git a/docs/wsl.md b/docs/wsl.md index 7c14a9de2f..6ca20c9fea 100644 --- a/docs/wsl.md +++ b/docs/wsl.md @@ -15,9 +15,9 @@ The AMD WSL path is: 1. Windows 11 with the AMD Adrenalin WSL-capable driver. 2. WSL2 with Ubuntu 24.04 or newer. Ubuntu 22.04 is not supported: it ships - glibc 2.35, below the minimum that every published Lemonade embeddable - requires (glibc 2.38 and `GLIBCXX_3.4.32`), so the Lemonade engine cannot - start there. Ubuntu 24.04 provides glibc 2.39 and `GLIBCXX_3.4.33`. + glibc 2.35, below what the Lemonade embeddable ROCm CLI pins requires + (glibc 2.38 and `GLIBCXX_3.4.32`; newer Lemonade releases need more), so + the Lemonade engine cannot start there. Ubuntu 24.04 provides glibc 2.39 and `GLIBCXX_3.4.33`. 3. ROCDXG (`librocdxg`) installed inside WSL. 4. A TheRock runtime installed by ROCm CLI into a managed Python venv. diff --git a/engines/lemonade/Cargo.toml b/engines/lemonade/Cargo.toml index 65e966f8ac..7c5ec45f06 100644 --- a/engines/lemonade/Cargo.toml +++ b/engines/lemonade/Cargo.toml @@ -41,7 +41,8 @@ zip = { version = "9.0", default-features = false, features = ["deflate"] } # Reads the version nodes the host's libc.so.6 and libssl.so.3 define, to # refuse a host older than the pinned `lemond` needs (`src/host_libraries.rs`). # 0.37 with only the ELF reader: `backtrace` already links this version into -# `rocm`, so it adds no crate to the shipped binary. +# `rocm`, so it adds no crate there. The standalone `rocm-engine-lemonade` +# binary has no `backtrace`, so for it `object` is one new crate. object = { version = "0.37", default-features = false, features = ["read_core", "elf"] } [dev-dependencies] From 391bea4642ffa89b4288271beef412c7a27acf3e Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 14:33:13 +0200 Subject: [PATCH 12/13] fix(lemonade): read the backend state at its column, and name the version node a host library provides A first-column warning with a state word in its text ("... is installed but outdated") started a bogus recipe and dropped the llama.cpp rows after it. The state is now read at the Status column the header names; with no header, only a recipe-shaped first word starts a recipe. The host-library check reports the newest version node a library defines, not its release: Ubuntu 24.04's OpenSSL 3.0.13 defines up to OPENSSL_3.0.3. The message now says "this host's libc.so.6 provides up to GLIBC_2.31", and test rows use values the probe can produce. Signed-off-by: Roman Inflianskas --- README.md | 3 +- docs/manual-testing.md | 3 +- engines/lemonade/src/host_libraries.rs | 60 +++++--- engines/lemonade/src/llamacpp_backend.rs | 139 ++++++++++++++---- engines/lemonade/src/process.rs | 5 +- .../tests/e2e/lemonade_install_steps.rs | 6 +- 6 files changed, 158 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 15e9f4f493..b71f673c96 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,8 @@ through 2026.44.1, the one rocm-cli pins, needs those versions; the newest, Ubuntu 22.04 ships glibc 2.35 and cannot run it; Ubuntu 24.04 provides glibc 2.39, `GLIBCXX_3.4.33` and OpenSSL 3.0.13. On an older host, `rocm engines install lemonade` stops before downloading anything and says what -falls short: the glibc version Lemonade needs and the one the host has, or that +falls short: the glibc version Lemonade needs and the newest the host's +`libc.so.6` provides, or that the host has no `libssl.so.3`, the library of OpenSSL 3. That check reads glibc and OpenSSL only; `GLIBCXX_3.4.32` (libstdc++) is not checked. Where Lemonade is already installed, the same check refuses to reinstall it or start it, and says diff --git a/docs/manual-testing.md b/docs/manual-testing.md index ce45f30007..1b3150ab4a 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -478,7 +478,8 @@ use a build with the `rocm/e2e-test-hooks` feature and set `ROCM_E2E_LEMONADE_HO `ROCM_E2E_LEMONADE_HOST_OPENSSL=missing` first. Expected result: - It fails with `this host's libraries are too old for the Lemonade - server: it needs glibc 2.38 or newer (this host has glibc )`, followed + server: it needs glibc 2.38 or newer (this host's libc.so.6 provides up to + GLIBC_)`, followed on a host without OpenSSL 3 by `and OpenSSL 3.0.0 or newer (this host's loader cache has no libssl.so.3)`, then `Nothing was downloaded.`, and ends by naming `ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK=1` as the way to skip diff --git a/engines/lemonade/src/host_libraries.rs b/engines/lemonade/src/host_libraries.rs index 3043f605cc..ff3ceda0e5 100644 --- a/engines/lemonade/src/host_libraries.rs +++ b/engines/lemonade/src/host_libraries.rs @@ -134,23 +134,34 @@ const OPENSSL_SONAME: &str = "libssl.so.3"; /// nothing short of the needs was found. /// /// Names each library that falls short with the version needed and what the -/// host has: its version, or that the loader cache has no library of that -/// soname. So the message alone says what to upgrade. +/// host has: the newest version node its library defines (which is what the +/// loader compares, and not always the release: Ubuntu 24.04's OpenSSL 3.0.13 +/// defines up to `OPENSSL_3.0.3`), or that the loader cache has no library of +/// that soname. So the message alone says what to upgrade. pub(crate) fn host_shortfall( lemonade_version: &str, needs: &LemondHostNeeds, host: &HostLibraries, ) -> Option { let mut short = Vec::new(); - for (name, soname, needed, have) in [ - ("glibc", GLIBC_SONAME, needs.glibc, host.glibc), - ("OpenSSL", OPENSSL_SONAME, needs.openssl, host.openssl), + for (name, soname, prefix, needed, have) in [ + ("glibc", GLIBC_SONAME, "GLIBC_", needs.glibc, host.glibc), + ( + "OpenSSL", + OPENSSL_SONAME, + "OPENSSL_", + needs.openssl, + host.openssl, + ), ] { let needed = LibVersion::parse(needed) .unwrap_or_else(|| panic!("pinned {name} need {needed:?} is not a version")); match have { + // `have` is the newest version node the library defines, not its + // release (Ubuntu 24.04's OpenSSL 3.0.13 defines up to + // `OPENSSL_3.0.3`), so the message names the node. HostLibrary::Version(have) if have < needed => short.push(format!( - "{name} {needed} or newer (this host has {name} {have})" + "{name} {needed} or newer (this host's {soname} provides up to {prefix}{have})" )), HostLibrary::Missing => short.push(format!( "{name} {needed} or newer (this host's loader cache has no {soname})" @@ -383,7 +394,7 @@ mod tests { fn versions_compare_numerically_and_ignore_trailing_zeros() { assert!(version("2.9") < version("2.38")); assert!(version("2.39") < version("2.43")); - assert!(version("3.0.13") < version("3.2.0")); + assert!(version("3.0.3") < version("3.2.0")); assert_eq!(version("3.0"), version("3.0.0")); assert_eq!(LibVersion::parse("2.2.5").unwrap().to_string(), "2.2.5"); for bad in ["", "2.", ".2", "2.x", "PRIVATE", "1.2.3.4.5", "-1"] { @@ -399,16 +410,18 @@ mod tests { #[test] fn shortfall_names_each_library_that_falls_short_and_what_the_host_has() { let rows: [(HostLibraries, Option<&str>); 7] = [ - (host(Some("2.39"), Some("3.0.13")), None), + (host(Some("2.39"), Some("3.0.3")), None), (host(Some("2.38"), Some("3.0.0")), None), ( host(Some("2.35"), Some("3.0.2")), - Some("it needs glibc 2.38 or newer (this host has glibc 2.35)"), + Some( + "it needs glibc 2.38 or newer (this host's libc.so.6 provides up to GLIBC_2.35)", + ), ), ( host(Some("2.31"), Some("missing")), Some( - "it needs glibc 2.38 or newer (this host has glibc 2.31) and \ + "it needs glibc 2.38 or newer (this host's libc.so.6 provides up to GLIBC_2.31) and \ OpenSSL 3.0.0 or newer (this host's loader cache has no libssl.so.3)", ), ), @@ -418,7 +431,7 @@ mod tests { "it needs OpenSSL 3.0.0 or newer (this host's loader cache has no libssl.so.3)", ), ), - (host(None, Some("3.0.13")), None), + (host(None, Some("3.0.3")), None), (host(None, None), None), ]; for (host, expected) in rows { @@ -440,7 +453,8 @@ mod tests { } /// An OpenSSL 3 older than a pin needs is reported by version: no pin needs - /// that today, but 2026.45.3 needed 3.2.0, which Ubuntu 24.04's 3.0.13 is not. + /// that today, but 2026.45.3 needed 3.2.0, and Ubuntu 24.04's OpenSSL 3.0.13 defines + /// only up to `OPENSSL_3.0.3`. #[test] fn an_openssl_3_older_than_the_need_is_named_by_version() { let needs = LemondHostNeeds { @@ -448,11 +462,11 @@ mod tests { glibc: "2.43", openssl: "3.2.0", }; - let message = host_shortfall("9.9.9", &needs, &host(Some("2.39"), Some("3.0.13"))).unwrap(); + let message = host_shortfall("9.9.9", &needs, &host(Some("2.39"), Some("3.0.3"))).unwrap(); assert!( message.ends_with( - "it needs glibc 2.43 or newer (this host has glibc 2.39) and \ - OpenSSL 3.2.0 or newer (this host has OpenSSL 3.0.13)" + "it needs glibc 2.43 or newer (this host's libc.so.6 provides up to GLIBC_2.39) and \ + OpenSSL 3.2.0 or newer (this host's libssl.so.3 provides up to OPENSSL_3.0.3)" ), "{message}" ); @@ -470,7 +484,7 @@ mod tests { panic!("a host short of glibc was not refused"); }; assert!( - message.contains("(this host has glibc 2.35). Nothing was downloaded. If this host"), + message.contains("(this host's libc.so.6 provides up to GLIBC_2.35). Nothing was downloaded. If this host"), "{message}" ); assert!( @@ -486,7 +500,7 @@ mod tests { host_check( "9.9.9", &NEEDS, - &host(Some("2.39"), Some("3.0.13")), + &host(Some("2.39"), Some("3.0.3")), "x", false ), @@ -495,7 +509,7 @@ mod tests { refuse_if_host_too_old( "9.9.9", &NEEDS, - &host(Some("2.39"), Some("3.0.13")), + &host(Some("2.39"), Some("3.0.3")), "x", false, ) @@ -513,7 +527,7 @@ mod tests { assert_eq!( warning, "warning: this host's libraries are too old for the Lemonade 9.9.9 server: it \ - needs glibc 2.38 or newer (this host has glibc 2.31) and OpenSSL 3.0.0 or newer \ + needs glibc 2.38 or newer (this host's libc.so.6 provides up to GLIBC_2.31) and OpenSSL 3.0.0 or newer \ (this host's loader cache has no libssl.so.3); going ahead because \ ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK is set." ); @@ -523,7 +537,7 @@ mod tests { host_check( "9.9.9", &NEEDS, - &host(Some("2.39"), Some("3.0.13")), + &host(Some("2.39"), Some("3.0.3")), "x", true ), @@ -537,7 +551,7 @@ mod tests { const LIBSSL11_LINE: &str = "\tlibssl.so.1.1 (libc6,x86-64) => /lib/x86_64-linux-gnu/libssl.so.1.1\n"; - /// A reader standing in for the ELF read: glibc 2.31 and OpenSSL 3.0.13, + /// A reader standing in for the ELF read: `GLIBC_2.31` and `OPENSSL_3.0.3`, /// or `None` for a library that does not parse. fn planted_reader(unparseable: &'static str) -> impl Fn(&str, &str) -> Option { move |path, _prefix| { @@ -546,7 +560,7 @@ mod tests { } else if path.ends_with("libc.so.6") { LibVersion::parse("2.31") } else { - LibVersion::parse("3.0.13") + LibVersion::parse("3.0.3") } } } @@ -571,7 +585,7 @@ mod tests { let both = format!("{header}{LIBC_LINE}{LIBSSL_LINE}"); assert_eq!( host_libraries_from_cache(Some(&both), planted_reader("-")), - host(Some("2.31"), Some("3.0.13")) + host(Some("2.31"), Some("3.0.3")) ); assert_eq!( host_libraries_from_cache(Some(&both), planted_reader("libssl.so.3")), diff --git a/engines/lemonade/src/llamacpp_backend.rs b/engines/lemonade/src/llamacpp_backend.rs index 54efdef375..3679d5c733 100644 --- a/engines/lemonade/src/llamacpp_backend.rs +++ b/engines/lemonade/src/llamacpp_backend.rs @@ -180,10 +180,11 @@ pub(crate) fn ensure_best_llamacpp_backend_with_parent( /// `src/cpp/server/backend_manager.cpp` at that tag). The other state Lemonade defines, `action_required` /// (used by other recipes), leaves the row out. /// -/// A row that starts in the first column and carries a state names its recipe; -/// an indented row continues the recipe above it with another backend. A -/// first-column line with no state, such as a warning, is skipped and leaves -/// the current recipe as it was. The recipe is read from +/// A row that starts in the first column and has a state in its Status column +/// names its recipe; an indented row continues the recipe above it with another +/// backend. Any other line, such as a warning, is skipped and leaves the current +/// recipe as it was, even when its text contains a state word: the state is read +/// at the column the header names, not from anywhere in the line. The recipe is read from /// the layout rather than matched against a list of known recipe names: each /// Lemonade release adds recipes (`llamacpp-hrx`, `ds4`, `thenoise` in /// 2026.44), and a row for a recipe missing from such a list used to be read as @@ -210,41 +211,83 @@ fn parse_llamacpp_backend_statuses(output: &str) -> Vec<(String, String)> { _ => None, } } + // A recipe name as Lemonade spells one: lower case, digits, `-` and `_`. + // Rules out a warning's first word (`Warning:`) in a table with no header. + fn is_recipe_name(token: &str) -> bool { + token + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'_') + } + // Each whitespace-separated token with the byte offset it starts at. + fn tokens_at(line: &str) -> Vec<(usize, &str)> { + let mut tokens = Vec::new(); + let mut start = None; + for (index, ch) in line.char_indices() { + match (ch.is_whitespace(), start) { + (true, Some(from)) => { + tokens.push((from, &line[from..index])); + start = None; + } + (false, None) => start = Some(index), + _ => {} + } + } + if let Some(from) = start { + tokens.push((from, &line[from..])); + } + tokens + } + // Where the header puts the Status column, once the header is read. + let mut status_column: Option = None; let mut current_recipe = String::new(); let mut result = Vec::new(); for line in output.lines() { - let tokens: Vec<&str> = line.split_whitespace().collect(); - let Some(&first) = tokens.first() else { + let tokens = tokens_at(line); + let Some(&(_, first)) = tokens.first() else { continue; }; - if first == "Recipe" || first.starts_with("---") { + if first == "Recipe" { + status_column = line.find("Status"); + continue; + } + if first.starts_with("---") { continue; } let continues_recipe = line.starts_with(char::is_whitespace); - let (backend, rest) = if continues_recipe { - (first, &tokens[1..]) - } else { - let (Some(backend), Some(rest)) = (tokens.get(1), tokens.get(2..)) else { - continue; - }; - // Only a row with a state in it names a recipe. Any other - // first-column line (a warning Lemonade printed mid-table) is - // skipped without ending the recipe above it, so the indented - // rows that follow still count for that recipe. - if !rest.iter().any(|token| is_state(token)) { - continue; - } - first.clone_into(&mut current_recipe); - (*backend, rest) + let backend_index = usize::from(!continues_recipe); + let Some(&(_, backend)) = tokens.get(backend_index) else { + continue; }; - if current_recipe != LLAMACPP_RECIPE { + // The row's state: with a header, the first token that starts at or + // after the Status column (a field wider than its column pushes the + // status right, never left); without one, the first state word after + // the backend. A line whose state slot holds no state is not a row, + // such as a warning Lemonade printed mid-table, even one with a state + // word in its text, and is skipped without ending the recipe above it, + // so the indented rows that follow still count for that recipe. + let state = match status_column { + Some(column) => tokens + .iter() + .skip(backend_index + 1) + .find(|(offset, _)| *offset >= column) + .map(|(_, token)| *token) + .filter(|token| is_state(token)), + None if continues_recipe || is_recipe_name(first) => tokens + .iter() + .skip(backend_index + 1) + .map(|(_, token)| *token) + .find(|token| is_state(token)), + None => None, + }; + let Some(state) = state else { continue; + }; + if !continues_recipe { + first.clone_into(&mut current_recipe); } - // The first state token is the Status column; a state word later in - // the row is message text. - let Some(state) = rest.iter().find(|token| is_state(token)) else { + if current_recipe != LLAMACPP_RECIPE { continue; - }; + } if let Some(status) = usable_status(state) { result.push((backend.to_owned(), status.to_owned())); } @@ -505,8 +548,10 @@ thinksound rocm experimental native installable Backend ); } - /// A first-column line with no state between a `llamacpp` row and its - /// indented rows is not a recipe: the rows after it are still `llamacpp`'s. + /// Lines that are not rows between a `llamacpp` row and its indented rows + /// do not end the `llamacpp` rows: a warning with no state word, one whose + /// text contains a state word (`installed`), and one whose first word looks + /// like a recipe name. With a header the state is read at its Status column. #[test] fn a_warning_line_inside_the_table_does_not_end_the_llamacpp_rows() { let table = "\ @@ -514,10 +559,32 @@ Recipe Backend Tier Format Status Message --------------------------------------------------------------------------------------------------------------- llamacpp cpu core native installable Backend is supported but not installed. lemonade backends install llamacpp:cpu Warning: could not reach GitHub to check for updates +Warning: llamacpp:rocm is installed but outdated +lemonade is installed but update_required for llamacpp:rocm rocm core native update_required Backend update is required before use. lemonade backends install llamacpp:rocm vulkan core native installed b11580 sd-cpp cpu core native installable Backend is supported but not installed. lemonade backends install sd-cpp:cpu rocm core native installable Backend is supported but not installed. lemonade backends install sd-cpp:rocm +"; + let expected = vec![ + ("cpu".to_owned(), "installable".to_owned()), + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "installed".to_owned()), + ]; + assert_eq!(parse_llamacpp_backend_statuses(table), expected); + } + + /// Without a header there is no Status column to read, so a first-column + /// line names a recipe only if its first word is shaped like one: a + /// `Warning:` line with a state word in it is still skipped. + #[test] + fn a_warning_with_a_state_word_does_not_end_the_llamacpp_rows_without_a_header() { + let table = "\ +llamacpp cpu installable Backend is supported but not installed. +Warning: llamacpp:rocm is installed but outdated + rocm update_required Backend update is required before use. + vulkan installed b11580 +sd-cpp cpu installable Backend is supported but not installed. "; assert_eq!( parse_llamacpp_backend_statuses(table), @@ -529,6 +596,20 @@ sd-cpp cpu core native installable Backend ); } + /// A field wider than its column pushes the Status column right; the state + /// is still the first token at or after the header's Status column. + #[test] + fn a_field_wider_than_its_column_still_finds_the_state() { + let table = "\ +Recipe Backend Tier Format Status Message/Version +llamacpp rocm core native-overflowing installed b11580 +"; + assert_eq!( + parse_llamacpp_backend_statuses(table), + vec![("rocm".to_owned(), "installed".to_owned())] + ); + } + #[test] fn another_recipes_backend_is_never_read_as_a_llamacpp_backend() { // Plain `lemonade backends` (no `--all`) leaves unsupported backends diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs index c07507f344..e68732967a 100644 --- a/engines/lemonade/src/process.rs +++ b/engines/lemonade/src/process.rs @@ -1009,7 +1009,10 @@ mod tests { .to_string(); let needs = &crate::install::LINUX_LEMOND_NEEDS; for expected in [ - format!("glibc {} or newer (this host has glibc 2.31)", needs.glibc), + format!( + "glibc {} or newer (this host's libc.so.6 provides up to GLIBC_2.31)", + needs.glibc + ), format!( "OpenSSL {} or newer (this host's loader cache has no libssl.so.3)", needs.openssl diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs index 5379c6e356..3ddc414af7 100644 --- a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -180,7 +180,7 @@ async fn assert_lemonade_install_refused_for_host_libraries(world: &mut E2eWorld "the install must fail on a host too old for the pinned Lemonade server:\n{output}" ); for expected in [ - format!("or newer (this host has glibc {OLD_HOST_GLIBC})"), + format!("or newer (this host's libc.so.6 provides up to GLIBC_{OLD_HOST_GLIBC})"), "or newer (this host's loader cache has no libssl.so.3)".to_owned(), "it needs glibc ".to_owned(), "Nothing was downloaded.".to_owned(), @@ -223,7 +223,7 @@ async fn assert_host_library_warning(world: &mut E2eWorld) { the `rocm/e2e-test-hooks` feature, which `cargo xtask e2e` enables:\n{output}" ); for expected in [ - format!("or newer (this host has glibc {OLD_HOST_GLIBC})"), + format!("or newer (this host's libc.so.6 provides up to GLIBC_{OLD_HOST_GLIBC})"), "going ahead because ROCM_CLI_DISABLE_LEMONADE_HOST_LIBRARY_CHECK is set.".to_owned(), ] { assert!( @@ -310,7 +310,7 @@ async fn assert_refused_and_installed_copy_kept(world: &mut E2eWorld) { runtime_dir.display() ); for expected in [ - format!("or newer (this host has glibc {OLD_HOST_GLIBC})"), + format!("or newer (this host's libc.so.6 provides up to GLIBC_{OLD_HOST_GLIBC})"), "Nothing was changed: the Lemonade ".to_owned(), expected, ] { From 9ee73386cd15eeac8788313446fb2e6e82efc57c Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 14:58:33 +0200 Subject: [PATCH 13/13] docs: list the new parser tests, drop unverifiable release-cadence figures, say who deletes the xfail rows Signed-off-by: Roman Inflianskas --- docs/testing.md | 5 ++++- engines/lemonade/src/backend_alignment.rs | 6 +++--- tests/e2e-cucumber/expectations.toml | 4 +++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index b948b22bc0..15f44b924b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2502,12 +2502,15 @@ Lemonade backend selection and alignment focused tests (reading the rows of their own, and the second alignment attempt choosing the newest `lemonade-sdk/llama.cpp` release that ships the archive for the active ROCm version rather than the newest release, and reading Lemonade's -`update_required`/`update_available` backend states): +`update_required`/`update_available` backend states, and skipping a warning +line inside the table, even one with a state word in it): ```bash cargo test -p rocm-engine-lemonade llamacpp_backend::tests::parse cargo test -p rocm-engine-lemonade llamacpp_backend::tests::another_recipes_backend cargo test -p rocm-engine-lemonade llamacpp_backend::tests::update_states +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::a_warning +cargo test -p rocm-engine-lemonade llamacpp_backend::tests::a_field_wider cargo test -p rocm-engine-lemonade backend_alignment::tests::tier2 cargo test -p rocm-engine-lemonade backend_alignment::tests::align_falls_through_to_tier2 cargo test -p rocm-engine-lemonade backend_alignment::tests::llamacpp_asset_name diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index 07bb3423b8..b7ccd91f2c 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -559,9 +559,9 @@ const fn llamacpp_backend_alignment_succeeded( const LLAMACPP_ROCM_STABLE_REPO: &str = "lemonade-sdk/llama.cpp"; /// How many of [`LLAMACPP_ROCM_STABLE_REPO`]'s newest releases Tier 2 searches -/// for one that ships the archive it needs. The repository publishes a build -/// about twice a day, so this covers roughly the last two weeks; one listing page -/// of this size is about 2 MB. +/// for one that ships the archive it needs: one page of GitHub's release listing, +/// so a single request. How far back that reaches depends on how often the +/// repository publishes, which this repo does not control. const LLAMACPP_RELEASES_SEARCHED: usize = 30; /// The `llamacpp:rocm-stable` archive Lemonade downloads for build `tag` paired diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index fc59d66931..2274bae5b0 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -275,7 +275,9 @@ flaky = true # (b11580) ships a ROCm 7.14 archive. So both rows may now XPASS. That has not # been observed yet: both scenarios are `@nightly`, which pull-request lanes do # not run. A stale XPASS on the first Windows nightly run after the 2026.44.1 pin -# lands means the cause is gone, and the row is deleted then. +# lands means the cause is gone, and the row is deleted then, in a follow-up by +# the author of the 2026.44.1 pin. ROCMAI-899 is already closed; the row stays +# until that nightly shows the XPASS. [["runtime-lemonade-backend-alignment-reported"]] when = { effective_engine = "lemonade", os = "windows" } bug = "ROCMAI-899"