Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
a5a724f
test(e2e): capture Lemonade backend identity on serve failures
jussielo-amd Nov 5, 2026
f475592
test(e2e): treat a flag after --engine as no engine in serve failure …
jussielo-amd Nov 5, 2026
f237d2d
test(e2e): keep the per-row flaky rule in the expectations header
jussielo-amd Nov 5, 2026
2649611
test(e2e): cover Linux hosts without a usable GPU in the #897 expecta…
jussielo-amd Nov 5, 2026
25e11c5
test(e2e): assert the report says why an oversized backend has no fin…
jussielo-amd Nov 5, 2026
f5876c5
Merge origin/main into the Lemonade serve failure evidence branch
jussielo-amd Nov 5, 2026
5f37981
docs(testing): state the flag-after---engine, oversized-binary and pr…
jussielo-amd Nov 5, 2026
c05b62c
test(e2e): compare backend candidate paths as paths, not printed text
jussielo-amd Nov 5, 2026
5463b83
Merge origin/main into the Lemonade serve failure evidence branch
jussielo-amd Nov 5, 2026
05bcc1c
fix(e2e): read the Lemonade runtime manifest through the path the eng…
jussielo-amd Nov 6, 2026
dfb844f
fix(e2e): run the llama-server probes after the failed service is sto…
jussielo-amd Nov 6, 2026
59d7891
fix(e2e): search deeper for llama-server candidates and say where an …
jussielo-amd Nov 6, 2026
f5bff5f
test(e2e): expect the depth-limit candidates in the order the search …
jussielo-amd Nov 6, 2026
b064945
fix(e2e): fingerprint llama-server builds up to 512 MiB instead of 32…
jussielo-amd Nov 6, 2026
465912f
test(e2e): run the probe timeout and kill path on every platform
jussielo-amd Nov 6, 2026
a9bf807
test(e2e): keep the revisit-before-deleting caveat in the expectation…
jussielo-amd Nov 6, 2026
11f338d
Merge origin/main into the Lemonade serve failure evidence branch
jussielo-amd Nov 6, 2026
1e514a9
fix(e2e): make the stop-ordering helper and its tests pass the CI cli…
jussielo-amd Nov 6, 2026
ee08e62
docs(testing): record that a current ROCm llama-server fits under the…
jussielo-amd Nov 6, 2026
9b07eca
fix(e2e): bound the candidate search by time and size, and keep it of…
jussielo-amd Nov 6, 2026
cde16c3
fix(e2e): do not say the search found nothing when a limit ended it e…
jussielo-amd Nov 6, 2026
f013eae
fix(e2e): find the helper child's pid when libtest runs it on one tes…
jussielo-amd Nov 6, 2026
490bbda
Merge origin/main into the Lemonade serve failure evidence branch
jussielo-amd Nov 6, 2026
d4ca134
fix(e2e): name the engine install manifest path so the registry write…
jussielo-amd Nov 6, 2026
8740210
fix(e2e): refuse symlinked files the report reads, not only symlinked…
jussielo-amd Nov 6, 2026
96f67f0
fix(e2e): keep the stop and the failure report when the Lemonade mani…
jussielo-amd Nov 9, 2026
2c5a106
fix(e2e): say so when a log grew past the cap after its size was read
jussielo-amd Nov 9, 2026
2840f96
test(e2e): point the Lemonade serve xfail rows at the open ticket
jussielo-amd Nov 9, 2026
4293a30
fix(e2e): refuse a symlink in the open itself, not in a check before it
jussielo-amd Nov 9, 2026
198e385
fix(e2e): report a truncated candidate list only when it is truncated…
jussielo-amd Nov 9, 2026
2e39826
test(e2e): do not fail the probe kill test when the helper child star…
jussielo-amd Nov 9, 2026
a577e0e
test(e2e): keep the helper child alive past the longest probe attempt
jussielo-amd Nov 9, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 24 additions & 0 deletions crates/rocm-core/src/paths.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,21 @@ impl AppPaths {
self.engine_dir(engine).join("manifests")
}

/// The manifest an engine's installer writes for its installed runtime.
///
/// One definition for the writer (the engine) and for every reader of the
/// file, so a reader cannot drift to a different name: a reader that
/// misses it degrades to "no manifest", which is indistinguishable from an
/// engine that was never installed.
///
/// This is the engine's own install manifest, not an entry in the runtime
/// registry (that is `apps/rocm`'s `runtime_manifest_path`), and the name keeps
/// the two apart: the registry has a text guard that treats any file spelling
/// that helper's name and writing a file as a second registry writer.
pub fn engine_install_manifest_path(&self, engine: &str) -> PathBuf {
self.engine_manifests_dir(engine).join("runtime.json")
}

pub fn engine_state_dir(&self, engine: &str) -> PathBuf {
self.engine_dir(engine).join("state")
}
Expand Down Expand Up @@ -355,6 +370,15 @@ mod tests {
paths.data_dir.join("automations").join("proposals.jsonl")
);
}
#[test]
fn engine_install_manifest_lives_in_the_engine_manifests_dir() {
let (_root, paths) = temp_app_paths("engine-runtime-manifest-path");
assert_eq!(
paths.engine_install_manifest_path("lemonade"),
paths.engine_manifests_dir("lemonade").join("runtime.json")
);
}

#[test]
fn with_managed_root_keeps_reprovisioning_flat_from_runtime_leaf() {
let data_root = PathBuf::from("/tmp/rocm-cli-reprovision");
Expand Down
107 changes: 106 additions & 1 deletion docs/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -1041,6 +1041,111 @@ Docker exercises AIM serve. The `e2e-gpu-nightly-strix` job logs whether Docker
is reachable ("Report whether Docker is reachable"); check that line before
relying on this coverage.

### Lemonade managed-serve failure evidence

On a failed serve, the shared serve steps preserve the full service log under
`service-logs/` in the E2E results artifact before stopping the service. The engine
is taken from `--engine X` or `--engine=X`, or else from the printed serve plan; if
neither names one, the report says `engine unknown: backend evidence skipped`. A
value that starts with `--` (as in `serve m --engine --managed`) is another flag, not
an engine, and counts as no engine named.
Lemonade failures also include the installed runtime manifest's version/backend, configured
`backend_versions.json`, candidate GPU `llama-server` paths, SHA-256 fingerprints
and `version.txt`,
and each candidate's `--version` and `--help` output. Candidates are searched
under the `rocm-stable`, `rocm-nightly`, `rocm` and `vulkan` directories of
`bin/llamacpp`, down to three directory levels below each; the cpu backend is not
searched. The search is bounded as a whole, not per directory: it reads at most
128 directories and 2048 directory entries across all four backend directories,
stops as soon as it finds a ninth candidate (it lists the first eight and says the
list is cut; exactly eight is not reported as cut), and stops when the
inventory's time is up, checking the clock at every directory and entry. Siblings are visited in
sorted order, so a search that finishes inside its limits is deterministic.
Symlinks are never followed: only real directories are entered and only a regular
file named `llama-server` counts, so a link cycle ends and a link out of the
backend directory is not read; the report says how many symlinked entries it
declined to follow. The same rule covers every file the report reads: the
manifest, the pins file, `version.txt` and a candidate's fingerprint target are
refused when the last path component is a symlink, and the report says `symlink
not followed` in place of the content. On unix the refusal is made by the open
itself (`O_NOFOLLOW`), so a path swapped for a link after an earlier check is still
refused; on Windows it is a metadata check followed by an open, so a swap between
the two is not caught (read-only evidence, and the swap needs write access to the
backend directory). Only the last component is checked: a link in a parent
directory is followed. When a limit or the clock ends the search early the report
says which, and does not claim an empty result. When nothing is found, the report
lists the searched locations and what `bin/llamacpp` holds (the first 16 names
out of at most 64 entries read, with a count of those omitted), and does not
guess at the cause: an interrupted download, a layout outside the search and a
build that was never installed look the same. The manifest and pin reads, the
search and each candidate's fingerprint run on blocking threads, never on an
async worker, and the search carries its own deadline because a timeout around a
blocking task would only drop its handle.
At most eight candidates
are hashed and probed, with two read-only probes per candidate. Each probe retains
up to 32 KiB of stdout and 32 KiB of stderr; only streams that discard bytes are
marked as truncated. Across sixteen probes this is up to 1 MiB of retained raw
output, plus metadata and formatting (invalid UTF-8 bytes may expand when rendered).
Each probe has a five-second timeout. The inventory has a 20-second budget that
starts when it is entered and covers the manifest and pin reads, the search and
the hashing; the probes have their own 20-second budget, which starts
when the service stop returns; hashing runs off the async executor, rejects files
over 512 MiB, and checks its deadline between 64 KiB reads, so memory stays at
one buffer whatever the size. The cap sits well above a dynamically linked
`llama-server`: the `rocm-stable` build b12109 on the Strix Halo Ubuntu lanes
was fingerprinted under the former 32 MiB cap, so current builds fit with room to
spare. The larger cap is margin for a statically linked multi-target build,
which is an assumption about future builds, not a measured size, and a larger
file is a finding in its own right. A candidate over the size cap gets no fingerprint, and
its entry says so with `sha256 unavailable: executable exceeds 512 MiB hash
limit`; it is still listed and probed. Filesystem
operations and process cleanup are not preemptible, so this is not a hard
wall-clock limit on the entire report.
A candidate or configured build pin is **not proof** of which binary the router
launched; a failed probe is retained as evidence, not treated as an identity.
Probes inherit the harness environment rather than reconstructing the engine's
managed library paths, so a probe loader error alone does not establish why the
serve failed. The report reads files first and executes a backend second. The
manifest, pins, candidate paths, fingerprints and `version.txt` are read once the
serve has exited non-zero or its readiness wait has timed out and the log has
been read and archived, but before the service is stopped, so they describe the state being
diagnosed. The `--version` and `--help` probes run after the stop returns, in the
report's "probes, run after the service stop was attempted" section: llama.cpp
builds commonly initialise a GPU backend even for `--version`, and doing that next
to a still-running router on a shared-memory device could change the state being
diagnosed. A stop that fails or finds no record can leave the router running; the
report's `stop` section says so, and the probes still run. No lane has shown
whether a probe initialises a GPU backend, which is why they are kept off the
router's side of the stop. The order is fixed in one function,
`backend_evidence_around_stop`, whose unit tests make the stop delete the candidate
to show the fingerprint was read before it and the probes came after it. Nothing
before the stop may keep it from running: a serve with no isolated scenario root
has no manifest to look for, so the report says `backend evidence unavailable:
the scenario has no isolated root` (and the stop's own status says it was not
attempted), and a panic while looking the manifest up is caught and reported the
same way, so the stop runs exactly once and the failure report is not lost.

The probe's timeout, kill and reap path is tested on every platform, Windows
included: the unit tests re-run the test binary itself as the child (an ignored
helper test selected by an environment variable), let the probe time out, and
require that the child is gone afterwards. The shell-script `llama-server`
stand-ins used by the other probe tests exist on unix only. Not covered anywhere
is the `kill/reap failed` branch, which needs an operating system that refuses to
kill a child it just started.

Use the evidence to distinguish backend startup, download, and router readiness
failures; ROCMAI-898 tracks the Strix Halo Ubuntu investigation (first tracked
as [issue #897](https://github.com/ROCm/rocm-cli/issues/897), which is closed). A
backend's ready line does not establish that the router serves the requested
model, and a readiness timeout alone does not establish a shutdown. Diagnostics do not change the CLI's asynchronous
managed-launch contract or imply a backend fix.

Exercise the diagnostics and expectation scoping without a GPU:

```bash
cargo test -p e2e-cucumber --lib lemonade
```

## Runtime Selection And Activation

List registered managed or read-only runtimes and their exact side-by-side keys:
Expand Down Expand Up @@ -2029,7 +2134,7 @@ exit) and `serve-40` in
(GPU lane: serving with no watcher enabled starts no helper; on a Linux host
that runs it with lemonade it is an expected failure in
[expectations.toml](../tests/e2e-cucumber/expectations.toml) while lemonade
serving on the Strix Halo Ubuntu lanes is broken (#897), so the helper check is
serving on the Strix Halo Ubuntu lanes is broken (ROCMAI-898), so the helper check is
not reached on those lanes. The Strix Halo Windows lane runs it and passes, and
the row also matches other Linux lemonade hosts, where a pass is tolerated).

Expand Down
39 changes: 38 additions & 1 deletion engines/lemonade/src/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ where
}

fn manifest_path(paths: &AppPaths) -> PathBuf {
paths.engine_manifests_dir(ENGINE_NAME).join("runtime.json")
paths.engine_install_manifest_path(ENGINE_NAME)
}

fn lemonade_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf {
Expand Down Expand Up @@ -573,6 +573,43 @@ mod tests {
#[cfg(windows)]
use std::process::{Command as ProcessCommand, Stdio};

/// The e2e harness reads this manifest back to describe a failed serve
/// (`tests/e2e-cucumber/src/lemonade_diagnostics.rs`), by the same path
/// helper and the field names asserted here. A rename on either side would
/// otherwise show up only as `unavailable:` in a report from the real lane.
#[test]
fn written_manifest_is_where_and_how_the_e2e_failure_report_reads_it() {
let dir = scratch_dir("manifest-path-and-fields");
let paths = AppPaths {
config_dir: dir.join("config"),
data_dir: dir.join("data"),
cache_dir: dir.join("cache"),
};
let manifest = LemonadeInstallManifest {
env_id: "lemonade-embeddable-fixture".to_owned(),
version: "fixture".to_owned(),
runtime_dir: dir.join("runtime"),
lemond: dir.join("runtime/lemond"),
lemonade: dir.join("runtime/lemonade"),
backend_recipe: LLAMACPP_RECIPE.to_owned(),
backend_name: ROCM_BACKEND_NAME.to_owned(),
installed_at_unix_ms: 1,
};
write_manifest(&paths, &manifest).unwrap();

let path = paths.engine_install_manifest_path("lemonade");
assert_eq!(manifest_path(&paths), path);
let written: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap();
assert_eq!(written["version"], "fixture");
assert_eq!(written["backend_recipe"], LLAMACPP_RECIPE);
assert_eq!(written["backend_name"], ROCM_BACKEND_NAME);
assert_eq!(
written["runtime_dir"],
serde_json::json!(manifest.runtime_dir)
);
fs::remove_dir_all(&dir).ok();
}

#[test]
fn embeddable_package_matches_runtime_os() {
let version = rocm_deps::LEMONADE_VERSION;
Expand Down
14 changes: 14 additions & 0 deletions scripts/xfail_expectations_hint.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,11 @@ def emit(hits: dict[str, list[str]]) -> None:
when = {}
bug = "#123"
reason = "example"

[["serve-lemonade-inference"]]
when = { effective_engine = "lemonade", os = "linux" }
bug = "ROCMAI-898"
reason = "example"
"""


Expand Down Expand Up @@ -188,6 +193,15 @@ def check(title="", body="", commits=""):
assert hits == {"EAI-7333": ["serve-vllm-inference"]}, hits
print("self-test: refs read from commit messages ok")

# A row keyed to an open tracker id follows that id, and an id that is no
# longer in any row (a closed issue the rows were repointed away from) does
# not match: the hint reads the ids out of the matrix, it hard-codes none.
hits = check(body="Refs ROCMAI-898")
assert hits == {"ROCMAI-898": ["serve-lemonade-inference"]}, hits
hits = check(body="Refs #897")
assert hits == {}, hits
print("self-test: hint follows the id in the matrix (ROCMAI-898 yes, #897 no) ok")

print("xfail hint self-test: ok")


Expand Down
3 changes: 2 additions & 1 deletion tests/e2e-cucumber/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,9 @@ rand.workspace = true
reqwest = { version = "0.13", features = ["json"] }
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
tempfile = "3"
tokio.workspace = true
tokio = { workspace = true, features = ["process", "io-util"] }
toml = "1.1"
# `ServeDir` backs the install scenario's loopback download server (see
# `src/loopback_http.rs`), so the E2E suite does not hand-roll HTTP framing,
Expand Down
Loading
Loading