From 907a561e293dcde6fb92f6f2770b15dfe1654cd2 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 1 Oct 2026 13:16:01 +0000 Subject: [PATCH 01/10] test(integrity): property-test artifact integrity and protocol parsing Adds property tests over the code that decides whether a downloaded artifact is authentic and the code that decodes untrusted engine output: the download digest gate, the Lemonade archive digest check, RSA signature verification, model selection and readiness, and the engine protocol types. The generators target the near-miss region rather than sampling uniformly: signatures mutated by one byte or re-encoded with leading zeros, digests differing by one nibble or sharing a correct prefix, PEMs with a BOM, stray whitespace or blank lines, loopback spellings padded or re-cased, and model ids with a variant suffix or one character off. A uniform generator never reaches the branches where these checks could fail open. Golden documents pin the wire spellings a round trip cannot see; arbitrary and mutated engine output is smoke-tested for panics. One behaviour the suites pin rather than assert away: readiness matches a bare family word (`qwen`, `llama`) against any model with a name token starting with it, in both directions. An existing unit test exercises one direction (asking for `qwen` is satisfied by a served `Qwen3-0.6B-GGUF`); the pinned consequence is the reverse, where an endpoint reporting only `qwen` reads as ready for other Qwen entries in the catalog too. Selection itself is exact up to case and surrounding whitespace; only the check that confirms the engine is serving the selected model is loose. Signed-off-by: Roman Inflianskas --- Cargo.lock | 1 + crates/rocm-core/Cargo.toml | 5 +- .../rocm-core/tests/digest_gate_properties.rs | 281 +++++ .../rocm-core/tests/integrity_properties.rs | 246 ++++ .../tests/model_selector_properties.rs | 586 ++++++++++ crates/rocm-engine-protocol/Cargo.toml | 4 + .../tests/protocol_properties.rs | 1013 +++++++++++++++++ engines/lemonade/src/archive_cache.rs | 118 ++ 8 files changed, 2253 insertions(+), 1 deletion(-) create mode 100644 crates/rocm-core/tests/digest_gate_properties.rs create mode 100644 crates/rocm-core/tests/integrity_properties.rs create mode 100644 crates/rocm-core/tests/model_selector_properties.rs create mode 100644 crates/rocm-engine-protocol/tests/protocol_properties.rs diff --git a/Cargo.lock b/Cargo.lock index b1e0dd392a..a68427e1b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3905,6 +3905,7 @@ dependencies = [ name = "rocm-engine-protocol" version = "0.1.0" dependencies = [ + "proptest", "rocm-core", "serde", "serde_json", diff --git a/crates/rocm-core/Cargo.toml b/crates/rocm-core/Cargo.toml index 48afe6fee4..cdb9855593 100644 --- a/crates/rocm-core/Cargo.toml +++ b/crates/rocm-core/Cargo.toml @@ -45,7 +45,10 @@ ureq = { version = "2.12", features = ["native-certs"] } # # Also used by `disk_space`, for the same reason: its unit formatting, margin # arithmetic and mount selection are small total functions whose contracts are -# easy to state over ALL inputs and awkward to cover with examples. +# easy to state over ALL inputs and awkward to cover with examples. And by the +# integration suites under `tests/` (download digest gate, signature +# verification, model selection and readiness), whose generators aim at near +# misses no example list enumerates. # Test-only, so it adds nothing to any shipped binary. proptest = { version = "1", default-features = false, features = ["std"] } # Archive-extraction property tests (`src/archive_props.rs`) build hostile diff --git a/crates/rocm-core/tests/digest_gate_properties.rs b/crates/rocm-core/tests/digest_gate_properties.rs new file mode 100644 index 0000000000..bb32bf17b4 --- /dev/null +++ b/crates/rocm-core/tests/digest_gate_properties.rs @@ -0,0 +1,281 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for the digest gate in `download_file_streaming`, the single +//! place a downloaded artifact's bytes are authenticated. +//! +//! The generator is built around *near misses*, because that is the only region +//! where this gate can plausibly fail open: a uniformly random 64-character +//! string differs from the real digest in ~every position and is rejected by +//! any implementation, correct or not. What distinguishes a correct comparison +//! from a prefix comparison, a truncating comparison, or a trimming one is a +//! candidate that shares most of the real digest. + +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::path::PathBuf; +use std::thread; +use std::time::Duration; + +use proptest::prelude::*; +use rocm_core::{DownloadRequest, download_file_streaming}; +use sha2::{Digest, Sha256}; + +/// Serve `body` once over HTTP on an ephemeral loopback port. +fn serve_once(body: Vec) -> (u16, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + let port = listener.local_addr().expect("local addr").port(); + let handle = thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let mut scratch = [0_u8; 2048]; + let _ = stream.read(&mut scratch); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(&body); + let _ = stream.flush(); + } + }); + (port, handle) +} + +fn scratch_dir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "rocm-core-digest-gate-{tag}-{}-{:?}", + std::process::id(), + thread::current().id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("scratch dir"); + dir +} + +/// Run one download against a one-shot server, returning whether the gate +/// accepted the body. +fn download_accepts(body: &[u8], expected_sha256: Option<&str>, tag: &str) -> bool { + download_outcome(body, expected_sha256, tag).0 +} + +/// Like [`download_accepts`], also returning every file the download left in +/// its otherwise empty scratch directory — the destination and any partial +/// file, whatever it is named. +fn download_outcome(body: &[u8], expected_sha256: Option<&str>, tag: &str) -> (bool, Vec) { + let dir = scratch_dir(tag); + let destination = dir.join("artifact.bin"); + let (port, server) = serve_once(body.to_vec()); + let url = format!("http://127.0.0.1:{port}/artifact.bin"); + let request = DownloadRequest { + expected_sha256, + ..DownloadRequest::new(&url, &destination, Duration::from_secs(10)) + }; + let accepted = download_file_streaming(&request).is_ok(); + let _ = server.join(); + let mut left_behind: Vec = std::fs::read_dir(&dir) + .expect("read scratch dir") + .map(|entry| { + entry + .expect("scratch entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + left_behind.sort(); + let _ = std::fs::remove_dir_all(&dir); + (accepted, left_behind) +} + +/// The chunk size the download streams through, copied from the private +/// `DOWNLOAD_CHUNK_BYTES` in `net.rs`. The multi-chunk body below spans three +/// of these plus a byte, so it still crosses a chunk boundary if that constant +/// grows a little, and regardless of how short the socket reads come back. +const CHUNK_BYTES: usize = 64 * 1024; + +fn digest_of(body: &[u8]) -> String { + format!("{:x}", Sha256::digest(body)) +} + +/// How to perturb the true digest. Each variant lands in the region where a +/// sloppy comparison would wrongly accept. +#[derive(Debug, Clone, Copy)] +enum NearMiss { + /// One hex nibble changed — the smallest possible difference. + OneNibble, + /// Correct prefix, wrong suffix: accepted by a prefix comparison. + WrongSuffix, + /// Correct prefix only, truncated: accepted by a `starts_with` comparison. + TruncatedPrefix, + /// Correct digest with a trailing newline, as a `.sha256` sidecar carries it. + TrailingNewline, + /// Correct digest with surrounding spaces. + Padded, + /// Correct digest with a 65th character appended. + Overlong, + /// Correct digest with a leading zero prepended (shifts every position). + LeadingZero, + /// Empty — the "no expectation" string, which must not read as a match. + Empty, + /// All-zero digest of the right shape. + AllZeroes, +} + +const ALL_NEAR_MISSES: [NearMiss; 9] = [ + NearMiss::OneNibble, + NearMiss::WrongSuffix, + NearMiss::TruncatedPrefix, + NearMiss::TrailingNewline, + NearMiss::Padded, + NearMiss::Overlong, + NearMiss::LeadingZero, + NearMiss::Empty, + NearMiss::AllZeroes, +]; + +fn apply_near_miss(digest: &str, miss: NearMiss, position: usize, keep: usize) -> String { + let position = position % digest.len(); + let keep = 1 + keep % (digest.len() - 1); + match miss { + NearMiss::OneNibble => { + let mut out: Vec = digest.chars().collect(); + out[position] = if out[position] == '0' { '1' } else { '0' }; + out.into_iter().collect() + } + NearMiss::WrongSuffix => { + let mut out = digest[..keep].to_owned(); + out.push_str(&"f".repeat(digest.len() - keep)); + if out == digest { + out.replace_range(keep..=keep, "0"); + } + out + } + NearMiss::TruncatedPrefix => digest[..keep].to_owned(), + NearMiss::TrailingNewline => format!("{digest}\n"), + NearMiss::Padded => format!(" {digest} "), + NearMiss::Overlong => format!("{digest}a"), + NearMiss::LeadingZero => format!("0{digest}"), + NearMiss::Empty => String::new(), + NearMiss::AllZeroes => "0".repeat(64), + } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 24, ..ProptestConfig::default() })] + + /// A near-miss digest must never be accepted. The one deliberate exception + /// is pure ASCII case folding, which the gate implements as equivalent and + /// which is covered separately below. + /// + /// Every kind of near miss is tried for every generated case, rather than + /// one kind drawn per case: a regression that admits just one kind would + /// otherwise go unnoticed in any run that happened not to draw it. + #[test] + fn near_miss_digests_are_rejected( + body in prop::collection::vec(any::(), 1..512), + position in any::(), + keep in any::(), + ) { + let truth = digest_of(&body); + for miss in ALL_NEAR_MISSES { + let candidate = apply_near_miss(&truth, miss, position, keep); + prop_assert!( + !candidate.eq_ignore_ascii_case(&truth), + "near-miss kind {miss:?} reproduced the true digest" + ); + prop_assert!( + !download_accepts(&body, Some(&candidate), "near-miss"), + "the digest gate accepted {candidate:?} for an artifact whose digest is {truth:?} \ + (near-miss kind {miss:?})" + ); + } + } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 12, ..ProptestConfig::default() })] + + /// The true digest is accepted, in either ASCII case, and in mixed case — + /// case-insensitivity must hold in both directions, not just for a fully + /// lowercase expectation. + #[test] + fn the_true_digest_is_accepted_in_any_ascii_case( + body in prop::collection::vec(any::(), 1..512), + mask in prop::collection::vec(any::(), 64), + ) { + let truth = digest_of(&body); + let mixed: String = truth + .chars() + .zip(mask) + .map(|(ch, upper)| if upper { ch.to_ascii_uppercase() } else { ch }) + .collect(); + prop_assert!(download_accepts(&body, Some(&mixed), "mixed-case")); + } + + /// Mutating the *body* while keeping the expected digest must be rejected, + /// and nothing — no destination, no partial file — may be left behind for a + /// later run to treat as cached. This checks the end state: the download + /// cleans up on more than one path, and either one suffices here. + #[test] + fn a_tampered_body_is_rejected( + body in prop::collection::vec(any::(), 2..512), + index in any::(), + bit in 0u8..8, + ) { + let truth = digest_of(&body); + let mut tampered = body; + let position = index.index(tampered.len()); + tampered[position] ^= 1 << bit; + let (accepted, left_behind) = download_outcome(&tampered, Some(&truth), "tampered"); + prop_assert!( + !accepted, + "a body differing by one bit was accepted against the original digest" + ); + prop_assert!( + left_behind.is_empty(), + "a rejected download left {left_behind:?} behind" + ); + } +} + +/// A body several streaming chunks long, tampered only in its last byte, is +/// rejected and leaves nothing behind, while the intact body is accepted: the +/// digest must cover every byte, not just the first buffer. (A hash over only a +/// prefix is caught by the intact half; the tampered half catches a gate that +/// is skipped or disabled for a body this large.) +#[test] +fn tampering_past_the_first_chunk_is_rejected() { + let body: Vec = (0..=255_u8).cycle().take(3 * CHUNK_BYTES + 1).collect(); + let truth = digest_of(&body); + assert!( + download_accepts(&body, Some(&truth), "multi-chunk-intact"), + "the untampered multi-chunk body must be accepted" + ); + let mut tampered = body; + *tampered.last_mut().expect("non-empty body") ^= 1; + let (accepted, left_behind) = download_outcome(&tampered, Some(&truth), "multi-chunk"); + assert!(!accepted, "a change in the last byte was not detected"); + assert!( + left_behind.is_empty(), + "a rejected download left {left_behind:?} behind" + ); +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 8, ..ProptestConfig::default() })] + + /// Without an expected digest the download succeeds — this is the gate's + /// documented fail-open mode, pinned here so it cannot be mistaken for + /// verification. `DownloadOutcome::sha256` is reported but nothing compares + /// it, so a caller that omits `expected_sha256` has authenticated nothing. + #[test] + fn an_absent_expectation_authenticates_nothing( + body in prop::collection::vec(any::(), 1..256), + ) { + prop_assert!( + download_accepts(&body, None, "no-expectation"), + "callers rely on a digest-less download succeeding; it just proves nothing" + ); + } +} diff --git a/crates/rocm-core/tests/integrity_properties.rs b/crates/rocm-core/tests/integrity_properties.rs new file mode 100644 index 0000000000..aade990e53 --- /dev/null +++ b/crates/rocm-core/tests/integrity_properties.rs @@ -0,0 +1,246 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for the signature-verification path that decides whether a +//! downloaded artifact or metadata document is authentic. +//! +//! The generators here deliberately sample the *near-miss* region — a signature +//! with one byte changed, a signature re-encoded with leading zeros, a payload +//! with one bit flipped, a PEM mangled by Windows tooling — because a uniformly +//! random generator essentially never produces an input that reaches the +//! interesting branches of an RSA verifier. + +use std::sync::OnceLock; + +use proptest::prelude::*; +use rocm_core::{ + generate_rsa_signing_keypair, sign_rsa_pkcs1_sha256_signature, + verify_rsa_pkcs1_sha256_signature, +}; + +/// One 2048-bit keypair, generated once: key generation takes seconds in a debug build, and every +/// property below only needs *a* valid key, not a fresh one per case. +fn keypair() -> &'static (String, String) { + static KEYPAIR: OnceLock<(String, String)> = OnceLock::new(); + KEYPAIR.get_or_init(|| generate_rsa_signing_keypair().expect("keypair generation")) +} + +/// A second, unrelated keypair, for "signed by the wrong key" cases. +fn other_keypair() -> &'static (String, String) { + static KEYPAIR: OnceLock<(String, String)> = OnceLock::new(); + KEYPAIR.get_or_init(|| generate_rsa_signing_keypair().expect("keypair generation")) +} + +fn sign(payload: &[u8]) -> Vec { + sign_rsa_pkcs1_sha256_signature(&keypair().0, payload).expect("signing") +} + +fn verify(payload: &[u8], signature: &[u8]) -> bool { + verify_rsa_pkcs1_sha256_signature(&keypair().1, payload, signature, "test").is_ok() +} + +/// Payloads shaped like the documents actually verified here — arbitrary bytes, +/// near-empty bodies, and printable text such as a metadata JSON blob — capped +/// at 512 bytes so shrinking stays fast. +fn payload_strategy() -> impl Strategy> { + prop_oneof![ + 3 => prop::collection::vec(any::(), 0..512), + 1 => prop::collection::vec(any::(), 0..4), + 1 => "[ -~\\n]{0,400}".prop_map(String::into_bytes), + ] +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 64, ..ProptestConfig::default() })] + + /// Baseline: a genuine signature over the exact payload verifies. + #[test] + fn genuine_signature_verifies(payload in payload_strategy()) { + let signature = sign(&payload); + prop_assert!(verify(&payload, &signature)); + } + + /// Flipping a single bit of the payload must invalidate the signature. + /// This is the "tampered artifact" case: the attacker edits one byte of a + /// signed metadata document and keeps the original signature. + #[test] + fn single_bit_payload_mutation_is_rejected( + payload in prop::collection::vec(any::(), 1..256), + index in any::(), + bit in 0u8..8, + ) { + let signature = sign(&payload); + let mut tampered = payload.clone(); + let position = index.index(payload.len()); + tampered[position] ^= 1 << bit; + prop_assert!( + !verify(&tampered, &signature), + "a payload differing by one bit must not verify" + ); + } + + /// Truncating or extending the payload must invalidate the signature, even + /// when the retained prefix is byte-identical. + #[test] + fn payload_prefix_or_extension_is_rejected( + payload in prop::collection::vec(any::(), 2..256), + cut in any::(), + suffix in prop::collection::vec(any::(), 1..16), + ) { + let signature = sign(&payload); + let keep = cut.index(payload.len()); + prop_assert!(!verify(&payload[..keep], &signature)); + + let mut extended = payload; + extended.extend_from_slice(&suffix); + prop_assert!(!verify(&extended, &signature)); + } + + /// Mutating the signature — one byte changed, truncated, padded, or + /// re-encoded with leading zeros — must be rejected. Leading-zero + /// re-encoding is the classic PKCS#1 v1.5 pitfall: the integer value is + /// unchanged, so a verifier that parses to a bignum without length-checking + /// accepts it. + #[test] + fn signature_mutations_are_rejected( + payload in payload_strategy(), + index in any::(), + delta in 1u8..=255, + pad in 1usize..4, + ) { + let signature = sign(&payload); + + let mut flipped = signature.clone(); + let position = index.index(flipped.len()); + flipped[position] = flipped[position].wrapping_add(delta); + prop_assert!(!verify(&payload, &flipped), "a mutated signature must not verify"); + + prop_assert!(!verify(&payload, &signature[..signature.len() - 1])); + prop_assert!(!verify(&payload, &signature[1..])); + prop_assert!(!verify(&payload, &[])); + + let mut zero_padded = vec![0_u8; pad]; + zero_padded.extend_from_slice(&signature); + prop_assert!( + !verify(&payload, &zero_padded), + "a leading-zero re-encoding carries the same integer value and must \ + still be rejected on length" + ); + + let mut trailing = signature.clone(); + trailing.extend(std::iter::repeat_n(0_u8, pad)); + prop_assert!(!verify(&payload, &trailing)); + } + + /// A signature produced by a different key must never verify. + #[test] + fn signature_from_another_key_is_rejected(payload in payload_strategy()) { + let wrong = sign_rsa_pkcs1_sha256_signature(&other_keypair().0, &payload) + .expect("signing with the second key"); + prop_assert!(!verify(&payload, &wrong)); + } + + /// Arbitrary bytes offered as a signature must be rejected, never accepted + /// and never panic. + #[test] + fn arbitrary_bytes_are_never_a_valid_signature( + payload in payload_strategy(), + candidate in prop::collection::vec(any::(), 0..300), + ) { + prop_assert!(!verify(&payload, &candidate)); + } +} + +/// PEM mangling that real tooling produces: CRLF, lone CR, a UTF-8 BOM, a stray +/// trailing space, a blank line, a tab. Verification must still succeed for +/// each. The BOM, space, blank-line and tab styles depend on `normalize_pem`; +/// CRLF and lone CR are also accepted by the PEM parser itself, so those two +/// styles pin the outcome rather than `normalize_pem`'s handling of them. +fn mangle_pem(pem: &str, style: u8) -> String { + match style % 6 { + 0 => pem.replace('\n', "\r\n"), + 1 => pem.replace('\n', "\r"), + 2 => format!("\u{feff}{pem}"), + 3 => pem.replace('\n', " \n"), + 4 => pem.replace('\n', "\n\n"), + _ => pem.replace('\n', "\t\r\n"), + } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 48, ..ProptestConfig::default() })] + + /// Tolerated mangling must not change *which* key is trusted. + #[test] + fn pem_mangling_preserves_the_verification_verdict( + payload in payload_strategy(), + style in any::(), + ) { + let signature = sign(&payload); + let mangled = mangle_pem(&keypair().1, style); + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&mangled, &payload, &signature, "test").is_ok(), + "whitespace/BOM mangling must not break a genuine signature" + ); + + // The same mangling applied to the *wrong* key must still reject. + let wrong = mangle_pem(&other_keypair().1, style); + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&wrong, &payload, &signature, "test").is_err(), + "normalization must not turn the wrong key into an accepting one" + ); + } + + /// Corrupting the base64 body of the public key must never yield a key that + /// accepts a signature made by the real key. + #[test] + fn corrupted_public_key_never_accepts( + payload in payload_strategy(), + index in any::(), + replacement in prop::sample::select( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + .chars() + .collect::>(), + ), + ) { + let signature = sign(&payload); + let pem = &keypair().1; + // Only the base64 body: everything after the header line and before + // the footer. + let body_start = pem.find('\n').expect("PEM header line") + 1; + let body_end = pem.find("-----END").expect("PEM footer"); + let body_positions: Vec = pem[body_start..body_end] + .char_indices() + .filter(|(_, ch)| ch.is_ascii_alphanumeric() || *ch == '+' || *ch == '/') + .map(|(position, _)| body_start + position) + .collect(); + let position = body_positions[index.index(body_positions.len())]; + let mut corrupted = pem.clone(); + corrupted.replace_range(position..=position, &replacement.to_string()); + if corrupted == *pem { + return Ok(()); + } + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&corrupted, &payload, &signature, "test").is_err(), + "a public key with a corrupted body must not accept the real signature" + ); + } + + /// Garbage offered as a public key must produce an error, never a success + /// and never a panic. + #[test] + fn arbitrary_public_key_text_is_rejected( + payload in payload_strategy(), + pem in prop_oneof![ + Just(String::new()), + "[ -~\\n]{0,200}", + "-----BEGIN PUBLIC KEY-----\n[A-Za-z0-9+/=\\n]{0,200}\n-----END PUBLIC KEY-----\n", + ], + ) { + let signature = sign(&payload); + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&pem, &payload, &signature, "test").is_err() + ); + } +} diff --git a/crates/rocm-core/tests/model_selector_properties.rs b/crates/rocm-core/tests/model_selector_properties.rs new file mode 100644 index 0000000000..522599b0f7 --- /dev/null +++ b/crates/rocm-core/tests/model_selector_properties.rs @@ -0,0 +1,586 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for model selection and for the readiness decision the CLI +//! makes from an engine's `/v1/models` response. +//! +//! Two separate risks live here. Selection must be *deterministic and +//! unambiguous*: `resolve_model_recipe` takes the first recipe whose +//! `matches_ref` returns true, so two recipes answering to the same selector +//! would make the winner an accident of catalog order. And readiness is decided +//! from a JSON document served by the engine — untrusted input — so it must +//! never panic and must not report a *different* model as the one the user +//! asked for. + +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::thread; +use std::time::Duration; + +use proptest::prelude::*; +use rocm_core::{ModelRecipeRecord, builtin_model_recipes, openai_models_endpoint_has_model}; + +fn catalog() -> &'static [ModelRecipeRecord] { + use std::sync::OnceLock; + static CATALOG: OnceLock> = OnceLock::new(); + CATALOG.get_or_init(builtin_model_recipes) +} + +/// Every selector the shipped catalog advertises: canonical ids and aliases. +fn advertised_selectors() -> Vec<(String, String)> { + let mut selectors = Vec::new(); + for recipe in catalog() { + selectors.push(( + recipe.canonical_model_id.clone(), + recipe.canonical_model_id.clone(), + )); + for alias in &recipe.aliases { + selectors.push((alias.clone(), recipe.canonical_model_id.clone())); + } + } + selectors +} + +/// Serve one HTTP response carrying `body` on an ephemeral loopback port. +fn serve_once(body: String) -> (u16, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + let port = listener.local_addr().expect("local addr").port(); + let handle = thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let mut scratch = [0_u8; 4096]; + let _ = stream.read(&mut scratch); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(body.as_bytes()); + let _ = stream.flush(); + } + }); + (port, handle) +} + +/// Ask the readiness check what it makes of `body` when expecting `expected`. +fn endpoint_reports(body: String, expected: Option<&str>) -> Result { + let (port, server) = serve_once(body); + let url = format!("http://127.0.0.1:{port}"); + let outcome = openai_models_endpoint_has_model(&url, expected, None, Duration::from_secs(10)) + .map_err(|error| error.to_string()); + let _ = server.join(); + outcome +} + +#[test] +fn the_shipped_catalog_is_not_empty() { + assert!( + !catalog().is_empty(), + "these properties are vacuous without a catalog to check" + ); +} + +/// No selector the catalog advertises may resolve to two different recipes. +/// `resolve_model_recipe` returns the first match, so an overlap means the +/// winner depends on JSON ordering rather than on the user's intent. +#[test] +fn no_advertised_selector_matches_two_recipes() { + let mut ambiguous: BTreeMap> = BTreeMap::new(); + for (selector, _) in advertised_selectors() { + let matches: Vec<&str> = catalog() + .iter() + .filter(|recipe| recipe.matches_ref(&selector)) + .map(|recipe| recipe.canonical_model_id.as_str()) + .collect(); + if matches.len() > 1 { + ambiguous.insert(selector, matches); + } + } + assert!( + ambiguous.is_empty(), + "these selectors resolve to more than one recipe, so the winner is catalog order: \ + {ambiguous:#?}" + ); +} + +/// Every advertised selector must resolve to the recipe that advertises it. +#[test] +fn every_advertised_selector_resolves_to_its_own_recipe() { + for (selector, owner) in advertised_selectors() { + let resolved: Vec<&str> = catalog() + .iter() + .filter(|recipe| recipe.matches_ref(&selector)) + .map(|recipe| recipe.canonical_model_id.as_str()) + .collect(); + assert_eq!( + resolved, + vec![owner.as_str()], + "selector {selector:?} must resolve to exactly its own recipe" + ); + } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 512, ..ProptestConfig::default() })] + + /// Selector matching ignores surrounding whitespace and ASCII case: any + /// re-casing of an advertised selector, padded with spaces, tabs or + /// newlines, still selects exactly its own recipe. (That other edits do + /// not match is the next property's job.) + #[test] + fn selector_matching_tolerates_only_case_and_surrounding_whitespace( + index in any::(), + upper in prop::collection::vec(any::(), 0..80), + lead in "[ \\t\\n]{0,3}", + trail in "[ \\t\\n]{0,3}", + ) { + let selectors = advertised_selectors(); + let (selector, owner) = &selectors[index.index(selectors.len())]; + let recased: String = selector + .chars() + .enumerate() + .map(|(position, ch)| { + if upper.get(position).copied().unwrap_or(false) { + ch.to_ascii_uppercase() + } else { + ch + } + }) + .collect(); + let decorated = format!("{lead}{recased}{trail}"); + let resolved: Vec<&str> = catalog() + .iter() + .filter(|recipe| recipe.matches_ref(&decorated)) + .map(|recipe| recipe.canonical_model_id.as_str()) + .collect(); + prop_assert_eq!( + resolved, + vec![owner.as_str()], + "case/whitespace decoration of {:?} changed which recipe it selects", + selector + ); + } + + /// Selection is otherwise exact: an advertised selector with one character + /// inserted (interior whitespace included) or deleted selects no recipe at + /// all — neither its own (no fuzzy matching) nor another — unless the edit + /// happens to spell some other advertised selector, up to ASCII case. + #[test] + fn a_mistyped_selector_selects_nothing( + index in any::(), + position in any::(), + delete in any::(), + inserted in prop::sample::select( + "abcdefghijklmnopqrstuvwxyz0123456789-_./ \t".chars().collect::>() + ), + ) { + let selectors = advertised_selectors(); + let (selector, _) = &selectors[index.index(selectors.len())]; + let mut mistyped: Vec = selector.chars().collect(); + if delete { + let at = position.index(mistyped.len()); + mistyped.remove(at); + } else { + let at = position.index(mistyped.len() + 1); + // Whitespace at either end is surrounding whitespace, which the + // previous property covers and selection deliberately ignores. + prop_assume!( + !(inserted.is_whitespace() && (at == 0 || at == mistyped.len())) + ); + mistyped.insert(at, inserted); + } + let mistyped: String = mistyped.into_iter().collect(); + prop_assume!( + !selectors + .iter() + .any(|(other, _)| other.eq_ignore_ascii_case(&mistyped)) + ); + + let selected: Vec<&str> = catalog() + .iter() + .filter(|recipe| recipe.matches_ref(&mistyped)) + .map(|recipe| recipe.canonical_model_id.as_str()) + .collect(); + prop_assert!( + selected.is_empty(), + "mistyped selector {:?} (from {:?}) selected {:?}", + mistyped, + selector, + selected + ); + } +} + +/// `/v1/models` bodies shaped the way a real engine shapes them, plus the +/// degenerate shapes a broken or hostile one could send. +fn models_body_strategy() -> impl Strategy { + prop_oneof![ + 3 => prop::collection::vec("[ -~]{0,24}", 0..4).prop_map(|ids| { + serde_json::json!({ + "object": "list", + "data": ids + .into_iter() + .map(|id| serde_json::json!({ "id": id, "object": "model" })) + .collect::>(), + }) + .to_string() + }), + 1 => Just("{}".to_owned()), + 1 => Just(r#"{"data": []}"#.to_owned()), + 1 => Just(r#"{"data": null}"#.to_owned()), + 1 => Just(r#"{"data": [{}]}"#.to_owned()), + 1 => Just(r#"{"data": [{"id": null}]}"#.to_owned()), + 1 => Just(r#"{"data": [[]]}"#.to_owned()), + 1 => Just(r#"{"data": "not-an-array"}"#.to_owned()), + 1 => Just("[]".to_owned()), + 1 => Just("null".to_owned()), + 1 => "[ -~]{0,64}", + ] +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 48, ..ProptestConfig::default() })] + + /// A malformed or hostile `/v1/models` document must produce a clean + /// `false` or a clean error — never a panic, and never a spurious "ready". + #[test] + fn a_models_document_never_reports_ready_without_naming_a_model( + body in models_body_strategy(), + expected in prop::sample::select(vec![ + "Qwen/Qwen3-4B-Instruct-2507", + "meta-llama/Llama-3.1-8B-Instruct", + "nonexistent/model", + ]), + ) { + // The call must return, not unwind. Any `Ok(true)` has to be justified + // by the document actually naming a model that matches. + if endpoint_reports(body.clone(), Some(expected)) == Ok(true) { + let value: serde_json::Value = serde_json::from_str(body.trim()) + .expect("a ready verdict requires a parseable document"); + let names_a_model = value + .get("data") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .any(|item| { + ["id", "model", "name"] + .into_iter() + .filter_map(|field| item.get(field).and_then(serde_json::Value::as_str)) + .any(|value| !value.trim().is_empty()) + }); + prop_assert!( + names_a_model, + "reported ready for {expected:?} from a document naming no model at all: {body}" + ); + } + } + + /// Positive control for the property above: a document that does list the + /// expected model — exactly, re-cased, padded, or by its basename alone, as + /// engines commonly report it — among other entries reads as ready. Without + /// this, a check that never reported ready would pass every negative + /// property here. + /// It pins the outcome, not which matching rule produces it: several rules + /// accept these spellings, so it does not stand guard over any one of them. + #[test] + fn a_models_document_listing_the_expected_model_reports_ready( + others in prop::collection::vec("[ -~]{0,24}", 0..3), + at in any::(), + spelling in 0u8..4, + expected in prop::sample::select(vec![ + "Qwen/Qwen3-4B-Instruct-2507", + "meta-llama/Llama-3.1-8B-Instruct", + "nonexistent/model", + ]), + ) { + let listed = match spelling { + 0 => expected.to_owned(), + 1 => expected.to_ascii_uppercase(), + 2 => format!(" {expected}\t"), + _ => expected.rsplit('/').next().unwrap_or(expected).to_owned(), + }; + let mut ids = others; + let position = at.index(ids.len() + 1); + ids.insert(position, listed.clone()); + let body = serde_json::json!({ + "object": "list", + "data": ids + .iter() + .map(|id| serde_json::json!({ "id": id, "object": "model" })) + .collect::>(), + }) + .to_string(); + prop_assert_eq!( + endpoint_reports(body, Some(expected)), + Ok(true), + "{:?} listed as {:?} was not reported ready", + expected, + listed + ); + } +} + +/// Whether `name` is a bare *family* word — a single alphabetic token of three +/// or more characters with no digit, like `qwen`, `llama` or `gemma`. +/// +/// Approximates the shape `rocm_core`'s private family matcher treats as a +/// family: readiness matches such a word against any model with a name token +/// that starts with it, so these names are excluded from the strict property +/// below and pinned separately. (Being an approximation, it only decides which +/// catalog names that property skips; the near-miss names that property also +/// checks never pass through it.) +fn is_bare_family_word(name: &str) -> bool { + let normalized: String = name + .trim() + .chars() + .filter(char::is_ascii_alphanumeric) + .flat_map(char::to_lowercase) + .collect(); + let single_token = name + .trim() + .split(|ch: char| !ch.is_ascii_alphanumeric()) + .filter(|token| !token.is_empty()) + .count() + == 1; + single_token && normalized.len() >= 3 && !normalized.chars().any(|ch| ch.is_ascii_digit()) +} + +/// An engine that lists only a *different* catalog model must not satisfy a +/// request for the one the user asked for. This is the "silently serving an +/// unrelated model" case: readiness is what gates the traffic the user then +/// sends, so a false positive routes inference to the wrong weights. +/// +/// Checked exhaustively over every ordered pair of catalog entries — including +/// each entry's aliases as the served id, since an engine echoes back whatever +/// name it was launched with — rather than sampled. The catalog is small, the +/// confusable pairs in it are specific (several `Qwen*` variants differing only +/// in size and quantization), and a sampling strategy can miss exactly those. +/// +/// Each wanted model is also checked against near misses of its own name that +/// it does not answer to: the name with a quantization or variant suffix, the +/// first name token containing a digit (`qwen3`, or just `3` for a Llama), a +/// two-letter stub, and the leading word with its first letter dropped +/// (`wen`). Those are what separate an exact comparison from a substring one, +/// and a family match from one that has lost its guards or matches inside a +/// word rather than at its start. +/// +/// Bare family words are excluded here and pinned by +/// [`a_bare_family_word_satisfies_readiness_for_the_whole_family`], which +/// documents the one shape that *does* cross-match today. +#[test] +fn listing_a_different_catalog_model_is_never_readiness() { + let recipes = catalog(); + assert!(recipes.len() > 1, "need at least two recipes to confuse"); + let mut false_positives = Vec::new(); + + for wanted in recipes { + for served in recipes { + if served.canonical_model_id == wanted.canonical_model_id { + continue; + } + let served_names = std::iter::once(&served.canonical_model_id).chain(&served.aliases); + for served_name in served_names { + // Skip a name the *wanted* recipe also answers to: that is a + // genuine shared alias, not a confusion. + if wanted.matches_ref(served_name) || is_bare_family_word(served_name) { + continue; + } + let body = serde_json::json!({ + "object": "list", + "data": [{ "id": served_name, "object": "model" }], + }) + .to_string(); + if endpoint_reports(body, Some(&wanted.canonical_model_id)) == Ok(true) { + false_positives.push(format!( + "wanted {:?} but an endpoint serving only {:?} reported ready", + wanted.canonical_model_id, served_name + )); + } + } + } + + for served_name in near_miss_names(&wanted.canonical_model_id) { + if wanted.matches_ref(&served_name) { + continue; + } + let body = serde_json::json!({ + "object": "list", + "data": [{ "id": served_name, "object": "model" }], + }) + .to_string(); + if endpoint_reports(body, Some(&wanted.canonical_model_id)) == Ok(true) { + false_positives.push(format!( + "wanted {:?} but an endpoint serving only the near miss {:?} reported ready", + wanted.canonical_model_id, served_name + )); + } + } + } + + assert!( + false_positives.is_empty(), + "readiness matched an unrelated model:\n{}", + false_positives.join("\n") + ); +} + +/// Readiness accepts a catalog model under any of its names: an engine that +/// reports an alias satisfies a request for the canonical id, and the reverse. +/// Checked for every alias in the shipped catalog. +#[test] +fn readiness_accepts_a_catalog_model_under_any_of_its_names() { + let mut misses = Vec::new(); + for recipe in catalog() { + for alias in &recipe.aliases { + for (served, wanted) in [ + (alias.as_str(), recipe.canonical_model_id.as_str()), + (recipe.canonical_model_id.as_str(), alias.as_str()), + ] { + let body = serde_json::json!({ + "object": "list", + "data": [{ "id": served, "object": "model" }], + }) + .to_string(); + if endpoint_reports(body, Some(wanted)) != Ok(true) { + misses.push(format!("serving {served:?} did not satisfy {wanted:?}")); + } + } + } + } + assert!(misses.is_empty(), "{}", misses.join("\n")); +} + +/// Names that sit next to `id` without being it: `id` with a variant suffix, +/// the first token of its basename that contains a digit, a two-letter stub of +/// its basename, and the basename's leading letters minus the first one. +fn near_miss_names(id: &str) -> Vec { + let basename = id.rsplit('/').next().unwrap_or(id); + let mut names: Vec = ["-FP8", "-GGUF", "-Instruct", "-v2"] + .iter() + .map(|suffix| format!("{id}{suffix}")) + .collect(); + if let Some(first_token) = basename + .split(|ch: char| !ch.is_ascii_alphanumeric()) + .find(|token| token.chars().any(|ch| ch.is_ascii_digit())) + { + names.push(first_token.to_ascii_lowercase()); + } + let stub: String = basename + .chars() + .filter(char::is_ascii_alphabetic) + .take(2) + .collect(); + if stub.len() == 2 { + names.push(stub.to_ascii_lowercase()); + } + let leading_word: String = basename + .chars() + .take_while(char::is_ascii_alphabetic) + .collect(); + if leading_word.len() >= 4 { + names.push(leading_word[1..].to_ascii_lowercase()); + } + names +} + +/// Characterization test for a known false-positive surface, pinned so it is +/// visible rather than merely absent from the suite. +/// +/// Readiness matches a bare *family* word against any model with a name token +/// that starts with it, and does so in both directions. One direction is +/// exercised by an existing unit test in `net.rs`: asking for `qwen` is +/// satisfied by an engine serving `Qwen3-0.6B-GGUF`. This test pins the other: an endpoint +/// whose `/v1/models` reports only the bare id `qwen` — itself a catalog alias +/// — is reported ready for other `Qwen*` entries in the catalog, not just the +/// one it actually loaded. The test asserts that such cross-matches exist and +/// that they stay inside the family. +/// +/// So when an engine reports nothing more specific than the family word, +/// readiness cannot tell the models of that family apart: a request for one +/// `Qwen*` model reads as ready whichever `Qwen*` model the engine has +/// resident. Note the asymmetry — *selection* (`matches_ref`) is exact up to +/// case and whitespace, while the readiness check that confirms the engine is +/// serving the selected model is looser. +/// +/// If readiness is ever tightened, this test is expected to fail and should be +/// updated along with the behavior. +#[test] +fn a_bare_family_word_satisfies_readiness_for_the_whole_family() { + let body = serde_json::json!({ + "object": "list", + "data": [{ "id": "qwen", "object": "model" }], + }) + .to_string(); + + let cross_matched: Vec<&str> = catalog() + .iter() + .map(|recipe| recipe.canonical_model_id.as_str()) + // Every catalog entry except the one that genuinely owns the `qwen` + // alias. + .filter(|id| { + !catalog() + .iter() + .find(|recipe| recipe.canonical_model_id == *id) + .is_some_and(|recipe| recipe.matches_ref("qwen")) + }) + .filter(|id| endpoint_reports(body.clone(), Some(id)) == Ok(true)) + .collect(); + + assert!( + !cross_matched.is_empty(), + "expected the documented family-word cross-match to still be present" + ); + assert!( + cross_matched + .iter() + .all(|id| id.to_ascii_lowercase().contains("qwen")), + "the cross-match must at least stay inside the family: {cross_matched:?}" + ); + + // The other direction, which the `net.rs` unit test also exercises: asking + // for the bare family word is satisfied by an engine serving any catalog + // model of that family. + for recipe in catalog().iter().filter(|recipe| { + recipe + .canonical_model_id + .to_ascii_lowercase() + .contains("qwen") + }) { + let body = serde_json::json!({ + "object": "list", + "data": [{ "id": recipe.canonical_model_id, "object": "model" }], + }) + .to_string(); + assert_eq!( + endpoint_reports(body, Some("qwen")), + Ok(true), + "asking for `qwen` was not satisfied by a served {:?}", + recipe.canonical_model_id + ); + } +} + +/// The same behavior outside the catalog: a user can `rocm serve` any Hugging +/// Face ref, and readiness compares the engine's reported id against it. A bare +/// family word therefore satisfies a request for any model of that family, +/// whatever its size. +#[test] +fn a_bare_family_word_satisfies_readiness_for_an_uncatalogued_model() { + for (served, wanted) in [ + ("qwen", "Qwen/Qwen3-235B-A22B-Instruct"), + ("llama", "meta-llama/Llama-3.1-70B-Instruct"), + ("gemma", "google/gemma-2-27b-it"), + ] { + let body = serde_json::json!({ + "object": "list", + "data": [{ "id": served, "object": "model" }], + }) + .to_string(); + assert_eq!( + endpoint_reports(body, Some(wanted)), + Ok(true), + "pinning the current behavior: serving {served:?} reads as ready for {wanted:?}" + ); + } +} diff --git a/crates/rocm-engine-protocol/Cargo.toml b/crates/rocm-engine-protocol/Cargo.toml index 19bd5f483a..1d60c6db3e 100644 --- a/crates/rocm-engine-protocol/Cargo.toml +++ b/crates/rocm-engine-protocol/Cargo.toml @@ -14,3 +14,7 @@ workspace = true rocm-core = { path = "../rocm-core" } serde.workspace = true serde_json.workspace = true + +[dev-dependencies] +# Property tests for the engine protocol decoders (`tests/protocol_properties.rs`). +proptest = { version = "1", default-features = false, features = ["std"] } diff --git a/crates/rocm-engine-protocol/tests/protocol_properties.rs b/crates/rocm-engine-protocol/tests/protocol_properties.rs new file mode 100644 index 0000000000..1bb43e3d3e --- /dev/null +++ b/crates/rocm-engine-protocol/tests/protocol_properties.rs @@ -0,0 +1,1013 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for the engine protocol: the types a plugin's stdout is +//! decoded into, and the helpers that turn untrusted filesystem and CLI input +//! into identifiers the CLI then acts on. +//! +//! Decoding of arbitrary bytes, arbitrary text and byte-level mutations of +//! valid encodings is smoke-tested for panics only: every type here uses +//! derived `Deserialize`, so those tests guard against a future hand-written +//! decoder or an unwrap on the decode path, not against wrong values. What a +//! decode must produce is pinned by the round-trip, version-skew and golden +//! document tests. + +use proptest::prelude::*; +use rocm_engine_protocol::{ + DetectResponse, DevicePolicy, EngineCapabilities, EngineDeviceAvailability, EngineMethod, + EngineRecipeEndpointHint, EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, + EngineRequestEnvelope, EngineResponseEnvelope, GpuSelection, HealthcheckResponse, + LaunchRequest, LogsRequest, LogsResponse, ResolveModelRequest, + engine_id_from_plugin_binary_name, gpu_indices_to_csv, is_public_bind_host, launch_gpu_indices, +}; +use std::collections::BTreeMap; + +/// Decode `bytes` as `T`, asserting only that it does not panic. +macro_rules! decode_without_panic { + ($type:ty, $bytes:expr) => { + let _ = serde_json::from_slice::<$type>($bytes); + }; +} + +/// Every envelope/payload type a plugin's output is decoded into. +fn decode_all_without_panic(bytes: &[u8]) { + decode_without_panic!(EngineRequestEnvelope, bytes); + decode_without_panic!(EngineResponseEnvelope, bytes); + decode_without_panic!(DetectResponse, bytes); + decode_without_panic!(HealthcheckResponse, bytes); + decode_without_panic!(LaunchRequest, bytes); + decode_without_panic!(LogsRequest, bytes); + decode_without_panic!(LogsResponse, bytes); + decode_without_panic!(ResolveModelRequest, bytes); + decode_without_panic!(EngineRecipeHint, bytes); + decode_without_panic!(GpuSelection, bytes); +} + +/// `any::()` deliberately excludes NaN and the infinities, so a naive +/// generator never samples the values JSON has no representation for — exactly +/// the region where a float round trip can break. Sample them explicitly. +fn tokens_per_sec_strategy() -> impl Strategy> { + proptest::option::of(prop_oneof![ + 6 => any::(), + 1 => Just(f32::NAN), + 1 => Just(f32::INFINITY), + 1 => Just(f32::NEG_INFINITY), + 1 => Just(f32::MIN_POSITIVE), + 1 => Just(-0.0_f32), + ]) +} + +fn healthcheck_strategy() -> impl Strategy { + ( + "[ -~]{0,20}", + any::(), + "[ -~]{0,20}", + any::(), + any::(), + proptest::option::of("[ -~]{0,30}"), + tokens_per_sec_strategy(), + ) + .prop_map( + |( + status, + model_loaded, + device, + uptime_sec, + queue_depth, + last_error, + tokens_per_sec, + )| { + HealthcheckResponse { + status, + model_loaded, + device, + uptime_sec, + queue_depth, + last_error, + tokens_per_sec, + } + }, + ) +} + +fn recipe_hint_strategy() -> impl Strategy { + ( + "[ -~]{0,10}", + "[ -~]{0,10}", + prop::collection::vec("[ -~]{0,12}", 0..4), + prop::collection::btree_map("[ -~]{0,8}", "[ -~]{0,8}", 0..4), + // Often empty: an empty map is omitted on the wire, so both the + // omitted and the present spelling have to round-trip. + prop::collection::btree_map("[ -~]{0,8}", "[ -~]{0,8}", 0..3), + proptest::option::of(( + "[ -~]{0,8}", + prop::collection::btree_map("[ -~]{0,8}", "[ -~]{0,8}", 0..3), + )), + prop::collection::vec(("[ -~]{0,8}", "[ -~]{0,8}"), 0..3), + prop::collection::vec("[ -~]{0,8}", 0..3), + ) + .prop_map( + |( + contract_version, + engine, + required_flags, + parser_settings, + extra_env, + preferred_endpoint, + unsupported, + notes, + )| EngineRecipeHint { + contract_version, + engine, + required_flags, + parser_settings, + extra_env, + preferred_endpoint: preferred_endpoint.map(|(endpoint_mode, settings)| { + EngineRecipeEndpointHint { + endpoint_mode, + settings, + } + }), + unsupported_combinations: unsupported + .into_iter() + .map( + |(combination, reason)| EngineRecipeUnsupportedCombinationHint { + combination, + reason, + }, + ) + .collect(), + notes, + }, + ) +} + +fn optional_text() -> impl Strategy> { + proptest::option::of("[ -~]{0,10}") +} + +fn detect_response_strategy() -> impl Strategy { + ( + ( + any::(), + optional_text(), + optional_text(), + optional_text(), + proptest::option::of(any::()), + optional_text(), + optional_text(), + optional_text(), + ), + prop::collection::vec( + ( + "[ -~]{0,8}", + any::(), + proptest::option::of("[ -~]{0,8}"), + ), + 0..4, + ), + ( + any::<[bool; 5]>(), + "[ -~]{0,8}", + prop::collection::vec("[ -~]{0,8}", 0..4), + ), + ) + .prop_map( + |( + ( + installed, + env_id, + runtime_kind, + runtime_executable, + managed_env, + python_version, + torch_version, + transformers_version, + ), + devices, + ( + [ + cpu, + rocm_gpu, + openai_compatible, + tool_calling, + reasoning_parser, + ], + quantized_models, + notes, + ), + )| DetectResponse { + installed, + env_id, + runtime_kind, + runtime_executable, + managed_env, + python_version, + torch_version, + transformers_version, + available_devices: devices + .into_iter() + .map(|(kind, available, reason)| EngineDeviceAvailability { + kind, + available, + reason, + }) + .collect(), + capabilities: EngineCapabilities { + cpu, + rocm_gpu, + openai_compatible, + tool_calling, + quantized_models, + reasoning_parser, + }, + notes, + }, + ) +} + +/// Byte-level mutations of a valid encoding: the only generator that reliably +/// reaches field-level decode logic. +fn mutate(json: &str, position: usize, kind: u8) -> Vec { + let mut bytes = json.as_bytes().to_vec(); + if bytes.is_empty() { + return bytes; + } + let position = position % bytes.len(); + match kind % 6 { + 0 => bytes[position] = b'"', + 1 => bytes[position] = b'{', + 2 => bytes[position] = 0xff, + 3 => { + bytes.truncate(position); + } + 4 => { + bytes.remove(position); + } + _ => bytes.insert(position, b'\\'), + } + bytes +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 512, ..ProptestConfig::default() })] + + /// Encode then decode must be the identity for every value the type holds. + #[test] + fn healthcheck_response_roundtrips(response in healthcheck_strategy()) { + let encoded = serde_json::to_string(&response).expect("encode"); + let decoded: HealthcheckResponse = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(decoded.status, response.status); + prop_assert_eq!(decoded.model_loaded, response.model_loaded); + prop_assert_eq!(decoded.device, response.device); + prop_assert_eq!(decoded.uptime_sec, response.uptime_sec); + prop_assert_eq!(decoded.queue_depth, response.queue_depth); + prop_assert_eq!(decoded.last_error, response.last_error); + // JSON has no representation for NaN or the infinities, so `serde_json` + // encodes them as `null` and they come back as `None`. Every *finite* + // value must round-trip exactly; a non-finite one is documented here as + // collapsing to "unknown" rather than surviving or erroring. + match response.tokens_per_sec { + Some(value) if !value.is_finite() => prop_assert_eq!( + decoded.tokens_per_sec, + None, + "a non-finite rate has no JSON representation and reads back as unknown" + ), + expected => prop_assert_eq!( + decoded.tokens_per_sec.map(f32::to_bits), + expected.map(f32::to_bits), + "a finite tokens_per_sec must survive an encode/decode round trip" + ), + } + } + + #[test] + fn recipe_hint_roundtrips(hint in recipe_hint_strategy()) { + let encoded = serde_json::to_string(&hint).expect("encode"); + let decoded: EngineRecipeHint = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(&decoded, &hint); + // Duplicate-serve reuse compares a recipe hint's JSON as a string, so an + // empty `extra_env` must stay off the wire: a hint encodes exactly as it + // did before the field existed. + let value = serde_json::to_value(&hint).expect("encode"); + prop_assert_eq!( + value.get("extra_env").is_some(), + !hint.extra_env.is_empty(), + "extra_env must be omitted exactly when it is empty" + ); + } + + #[test] + fn detect_response_roundtrips(response in detect_response_strategy()) { + let encoded = serde_json::to_string(&response).expect("encode"); + let decoded: DetectResponse = serde_json::from_str(&encoded).expect("decode"); + // `DetectResponse` has no `PartialEq`; compare the whole struct through + // its JSON value instead of a hand-picked subset of fields. (Being a + // round trip, this cannot see a renamed field.) + prop_assert_eq!( + serde_json::to_value(&decoded).expect("re-encode"), + serde_json::to_value(&response).expect("encode") + ); + } + + /// `EngineResponseEnvelope::success` serializes caller data with `expect`; + /// it must not panic for the response types the engines actually send, and + /// the data must arrive intact on the other side of the wire. + #[test] + fn response_envelope_success_carries_the_data_intact(response in healthcheck_strategy()) { + let envelope = EngineResponseEnvelope::success(&response); + prop_assert!(envelope.ok); + let encoded = serde_json::to_string(&envelope).expect("encode"); + let decoded: EngineResponseEnvelope = serde_json::from_str(&encoded).expect("decode"); + prop_assert!(decoded.ok); + prop_assert!(decoded.error.is_none()); + // Compared as the response it decodes back into, re-encoded: `serde_json` + // parses floats to the nearest f64 only approximately, so comparing the + // raw `Value`s would report a parser rounding as a lost field. + let data = decoded.data.expect("a success envelope carries data"); + let carried: HealthcheckResponse = serde_json::from_value(data).expect("data decodes"); + prop_assert_eq!( + serde_json::to_string(&carried).expect("re-encode"), + serde_json::to_string(&response).expect("encode"), + "the envelope must carry the caller's data unchanged" + ); + } + + /// Smoke test: mutating a valid encoding must yield either a clean decode + /// or a clean error — never a panic. This can only fail on a panic. + #[test] + fn mutated_valid_encodings_never_panic( + response in healthcheck_strategy(), + hint in recipe_hint_strategy(), + position in any::(), + kind in any::(), + ) { + for json in [ + serde_json::to_string(&response).expect("encode"), + serde_json::to_string(&hint).expect("encode"), + serde_json::to_string(&EngineResponseEnvelope::success(&response)).expect("encode"), + ] { + decode_all_without_panic(&mutate(&json, position, kind)); + } + } + + /// Fully arbitrary bytes must never panic a decoder. + #[test] + fn arbitrary_bytes_never_panic(bytes in prop::collection::vec(any::(), 0..256)) { + decode_all_without_panic(&bytes); + } + + #[test] + fn arbitrary_text_never_panics(text in "[ -~\\n\\t]{0,256}") { + decode_all_without_panic(text.as_bytes()); + } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 1024, ..ProptestConfig::default() })] + + /// A plugin binary name is attacker-influenced (anyone who can drop a file + /// into a plugin directory). The derived engine id is later used to build + /// paths and command lines, so it must never carry a separator or a + /// `.`/`..` traversal component, and the helper must never panic on a + /// multi-byte name. + #[test] + fn engine_id_is_never_a_traversal_component( + name in prop_oneof![ + "[ -~]{0,24}", + "rocm-engine-[ -~]{0,16}", + "rocm-engine-[A-Za-z0-9._/\\\\-]{0,12}(\\.exe|\\.EXE|)", + "\\PC{0,16}", + // The traversal names themselves, which no random strategy above + // reaches: an id of exactly `.` or `..`, with and without `.exe`. + prop::sample::select(vec![ + "rocm-engine-.", + "rocm-engine-..", + "rocm-engine-..exe", + "rocm-engine-...exe", + "rocm-engine-.EXE", + ]) + .prop_map(str::to_owned), + ], + ) { + if let Some(id) = engine_id_from_plugin_binary_name(&name) { + prop_assert!(!id.is_empty()); + prop_assert!(!id.contains('/'), "engine id must not contain `/`: {id:?}"); + prop_assert!(!id.contains('\\'), "engine id must not contain `\\`: {id:?}"); + prop_assert!(id != "." && id != "..", "engine id must not traverse: {id:?}"); + prop_assert!( + id.chars().all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-')), + "engine id must stay in the documented character set: {id:?}" + ); + } + } + + /// `--gpu` is direct user input. Parsing must never panic, and a successful + /// parse must round-trip through the CLI spelling it reports. + #[test] + fn gpu_selection_parse_never_panics_and_roundtrips(value in "[ -~]{0,16}") { + match GpuSelection::parse_cli_value(&value) { + Ok(GpuSelection::Auto) => { + prop_assert!(launch_gpu_indices(Some(&GpuSelection::Auto)).is_empty()); + } + Ok(GpuSelection::Index(index)) => { + prop_assert_eq!( + GpuSelection::parse_cli_value(&index.to_string()), + Ok(GpuSelection::Index(index)) + ); + prop_assert_eq!(launch_gpu_indices(Some(&GpuSelection::Index(index))), vec![index]); + prop_assert_eq!( + gpu_indices_to_csv(&[index]), + Some(index.to_string()) + ); + } + Err(_) => {} + } + } + + /// A GPU ordinal must never reach `HIP_VISIBLE_DEVICES` as anything but a + /// plain comma-separated list of decimal integers — the value is + /// interpolated into a child process environment. + #[test] + fn gpu_csv_is_always_plain_decimal(indices in prop::collection::vec(any::(), 0..8)) { + match gpu_indices_to_csv(&indices) { + None => prop_assert!(indices.is_empty()), + Some(csv) => { + prop_assert!(!indices.is_empty()); + let parsed: Result, _> = + csv.split(',').map(str::parse::).collect(); + prop_assert_eq!(parsed.as_deref(), Ok(indices.as_slice())); + } + } + } + + /// The loopback allowlist decides whether an endpoint gets an API key. + /// Anything that is not one of the three exact accepted spellings must be + /// classified public (key required) — misclassifying a public bind as + /// loopback opens an unauthenticated listener. + /// + /// A random host string essentially never lands near those spellings, so + /// most cases are drawn from the near-miss region: each loopback spelling + /// padded, re-cased, bracketed or extended, plus other addresses that + /// reach this host but are not on the list. + #[test] + fn only_exact_loopback_spellings_skip_the_api_key( + host in prop_oneof![ + 3 => ( + prop::sample::select(LOOPBACK_SPELLINGS.to_vec()), + 0u8..8, + ) + .prop_map(|(spelling, kind)| near_loopback(spelling, kind)), + 1 => prop::sample::select(OTHER_LOCAL_ADDRESSES.to_vec()).prop_map(str::to_owned), + 1 => "[ -~]{0,24}", + ], + ) { + let loopback = !is_public_bind_host(&host); + let allowlisted = LOOPBACK_SPELLINGS.contains(&host.as_str()); + prop_assert!( + loopback == allowlisted, + "host {host:?} was classified {} but only the exact spellings {LOOPBACK_SPELLINGS:?} \ + may skip the key", + if loopback { "loopback" } else { "public" } + ); + } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 256, ..ProptestConfig::default() })] + + /// A `LogsRequest` carries a `tail_lines` count straight from the caller into + /// the engines' tail logic; it must round-trip exactly, including the + /// extremes an engine could legitimately be asked for. + #[test] + fn logs_request_roundtrips( + service_id in "[ -~]{0,24}", + tail_lines in proptest::option::of(prop_oneof![ + Just(0usize), + Just(usize::MAX), + 0usize..10_000, + ]), + ) { + let request = LogsRequest { service_id: service_id.clone(), tail_lines }; + let encoded = serde_json::to_string(&request).expect("encode"); + let decoded: LogsRequest = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(decoded.service_id, service_id); + prop_assert_eq!(decoded.tail_lines, tail_lines); + } + + #[test] + fn logs_response_roundtrips( + log_path in "[ -~]{0,24}", + lines in prop::collection::vec("\\PC{0,40}", 0..8), + ) { + let response = LogsResponse { + log_path: log_path.clone(), + recent_lines: lines.clone(), + }; + let encoded = serde_json::to_string(&response).expect("encode"); + let decoded: LogsResponse = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(decoded.log_path, log_path); + prop_assert_eq!(decoded.recent_lines, lines); + } + + /// `host` and `port` from a launch request are what the CLI later classifies + /// for API-key enforcement, so they must survive the wire unchanged. + #[test] + fn launch_request_roundtrips_host_and_port( + host in "[ -~]{0,24}", + port in any::(), + selection in proptest::option::of(prop_oneof![ + Just(GpuSelection::Auto), + any::().prop_map(GpuSelection::Index), + ]), + device_policy in proptest::option::of(prop_oneof![ + Just(DevicePolicy::GpuRequired), + Just(DevicePolicy::GpuPreferred), + Just(DevicePolicy::CpuOnly), + ]), + ) { + let request = LaunchRequest { + service_id: "svc".to_owned(), + env_id: None, + runtime_id: None, + model_ref: "model".to_owned(), + host: host.clone(), + port, + device_policy: device_policy.clone(), + endpoint_mode: None, + engine_recipe: None, + gpu_selection: selection.clone(), + }; + let encoded = serde_json::to_string(&request).expect("encode"); + let decoded: LaunchRequest = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(&decoded.host, &host); + prop_assert_eq!(decoded.port, port); + prop_assert_eq!(decoded.gpu_selection, selection); + prop_assert_eq!(decoded.device_policy, device_policy); + prop_assert_eq!( + is_public_bind_host(&decoded.host), + is_public_bind_host(&host), + "the bind classification must not change across the wire" + ); + } + + /// A request envelope decodes exactly when its method is one of the + /// documented wire names — an unknown method is rejected rather than + /// falling back to some default — and a decoded method re-encodes to the + /// spelling it arrived as. Half the cases use a real method name, the rest + /// near misses of one or arbitrary words. + #[test] + fn request_envelope_accepts_exactly_the_known_methods( + method in prop_oneof![ + 2 => prop::sample::select(METHOD_WIRE_NAMES.to_vec()).prop_map(str::to_owned), + 1 => (prop::sample::select(METHOD_WIRE_NAMES.to_vec()), 0u8..4).prop_map( + |(name, kind)| match kind { + 0 => name.to_ascii_uppercase(), + 1 => format!("{name}s"), + 2 => name.replace('_', ""), + _ => format!(" {name}"), + } + ), + 1 => "[a-z_]{0,16}", + ], + payload in "[ -~]{0,24}", + ) { + let json = serde_json::json!({ "method": method, "payload": payload }).to_string(); + let parsed = serde_json::from_str::(&json); + let known = METHOD_WIRE_NAMES.contains(&method.as_str()); + prop_assert_eq!( + parsed.is_ok(), + known, + "method {:?} decoded: {}", + method, + parsed.is_ok() + ); + if let Ok(envelope) = parsed { + let reencoded = serde_json::to_value(&envelope).expect("encode"); + prop_assert_eq!( + reencoded.get("method").and_then(serde_json::Value::as_str), + Some(method.as_str()), + "a decoded method must re-encode to the same wire spelling" + ); + } + } +} + +/// The documented wire name of every engine method, written out rather than +/// derived from the enum. +const METHOD_WIRE_NAMES: [&str; 9] = [ + "detect", + "install", + "capabilities", + "resolve_model", + "launch", + "healthcheck", + "endpoint", + "stop", + "logs", +]; + +/// The three spellings the CLI accepts as loopback, written out here rather than +/// shared with the code under test. +const LOOPBACK_SPELLINGS: [&str; 3] = ["127.0.0.1", "localhost", "::1"]; + +/// Addresses that reach this host, or every interface, without being one of the +/// accepted spellings: all must be treated as public. +const OTHER_LOCAL_ADDRESSES: [&str; 9] = [ + "0.0.0.0", + "::", + "[::1]", + "::ffff:127.0.0.1", + "127.0.0.2", + "127.000.0.1", + "0:0:0:0:0:0:0:1", + "localhost.localdomain", + "", +]; + +/// One near miss of a loopback spelling; kind 0 is the spelling itself. +fn near_loopback(spelling: &str, kind: u8) -> String { + match kind { + 0 => spelling.to_owned(), + 1 => format!(" {spelling}"), + 2 => format!("{spelling}\n"), + 3 => spelling.to_ascii_uppercase(), + 4 => { + let mut chars = spelling.chars(); + chars + .next() + .map(|first| first.to_ascii_uppercase().to_string() + chars.as_str()) + .unwrap_or_default() + } + 5 => format!("[{spelling}]"), + 6 => format!("{spelling}:8080"), + _ => format!("{spelling}."), + } +} + +/// Present four times in five, so that a meaningful share of documents carry +/// every required field and take the decode-succeeds path. +fn mostly_present( + value: impl Strategy + 'static, +) -> impl Strategy> { + proptest::option::weighted(0.8, value) +} + +/// A map strategy over the fields of a valid response, so that fields can be +/// dropped individually — version skew as an engine would produce it, not as +/// random bytes would. +fn field_skew_strategy() -> impl Strategy { + ( + mostly_present("[ -~]{0,8}"), + mostly_present(any::()), + mostly_present("[ -~]{0,8}"), + mostly_present(any::()), + mostly_present(any::()), + "[ -~]{0,8}", + ) + .prop_map( + |(status, model_loaded, device, uptime, queue_depth, extra)| { + let mut object = serde_json::Map::new(); + if let Some(status) = status { + object.insert("status".to_owned(), serde_json::Value::String(status)); + } + if let Some(model_loaded) = model_loaded { + object.insert( + "model_loaded".to_owned(), + serde_json::Value::Bool(model_loaded), + ); + } + if let Some(device) = device { + object.insert("device".to_owned(), serde_json::Value::String(device)); + } + if let Some(uptime) = uptime { + object.insert("uptime_sec".to_owned(), serde_json::Value::from(uptime)); + } + if let Some(queue_depth) = queue_depth { + object.insert( + "queue_depth".to_owned(), + serde_json::Value::from(queue_depth), + ); + } + object.insert( + "unknown_future_field".to_owned(), + serde_json::Value::String(extra), + ); + serde_json::Value::Object(object) + }, + ) +} + +/// The fields `HealthcheckResponse` cannot decode without. +const HEALTHCHECK_REQUIRED_FIELDS: [&str; 5] = [ + "status", + "model_loaded", + "device", + "uptime_sec", + "queue_depth", +]; + +proptest! { + #![proptest_config(ProptestConfig { cases: 256, ..ProptestConfig::default() })] + + /// Version skew: a newer engine sends extra fields and may omit ones this + /// build requires. The document decodes exactly when every required field + /// is present — the unknown field it always carries must be ignored — and + /// a decoded response carries the values that were sent. + #[test] + fn version_skew_decodes_exactly_when_required_fields_are_present(value in field_skew_strategy()) { + let bytes = serde_json::to_vec(&value).expect("encode"); + let all_required_present = HEALTHCHECK_REQUIRED_FIELDS + .iter() + .all(|field| value.get(field).is_some()); + let decoded = serde_json::from_slice::(&bytes); + prop_assert_eq!( + decoded.is_ok(), + all_required_present, + "decode result {:?} for {}", + decoded.as_ref().map(|_| ()).map_err(ToString::to_string), + value + ); + if let Ok(response) = decoded { + prop_assert_eq!(Some(response.status.as_str()), value["status"].as_str()); + prop_assert_eq!(Some(response.model_loaded), value["model_loaded"].as_bool()); + prop_assert_eq!(Some(response.device.as_str()), value["device"].as_str()); + prop_assert_eq!(Some(response.uptime_sec), value["uptime_sec"].as_u64()); + prop_assert_eq!( + Some(u64::from(response.queue_depth)), + value["queue_depth"].as_u64() + ); + prop_assert_eq!(response.last_error, None); + } + decode_all_without_panic(&bytes); + } +} + +/// Golden documents for the wire spellings the round trips cannot see: device +/// policy and GPU selection values, and a literal healthcheck response as an +/// engine binary built from another revision would send it. +#[test] +fn protocol_values_use_their_documented_wire_spellings() { + for (policy, wire) in [ + (DevicePolicy::GpuRequired, "gpu_required"), + (DevicePolicy::GpuPreferred, "gpu_preferred"), + (DevicePolicy::CpuOnly, "cpu_only"), + ] { + assert_eq!( + serde_json::to_value(&policy).expect("encode"), + serde_json::json!(wire) + ); + } + assert_eq!( + serde_json::to_value(GpuSelection::Auto).expect("encode"), + serde_json::json!("auto") + ); + assert_eq!( + serde_json::to_value(GpuSelection::Index(1)).expect("encode"), + serde_json::json!({ "index": 1 }) + ); + + let response: HealthcheckResponse = serde_json::from_value(serde_json::json!({ + "status": "ready", + "model_loaded": true, + "device": "gpu", + "uptime_sec": 12, + "queue_depth": 3, + "last_error": "boom", + "tokens_per_sec": 1.5, + })) + .expect("a healthcheck document decodes"); + assert_eq!(response.status, "ready"); + assert!(response.model_loaded); + assert_eq!(response.device, "gpu"); + assert_eq!(response.uptime_sec, 12); + assert_eq!(response.queue_depth, 3); + assert_eq!(response.last_error.as_deref(), Some("boom")); + assert_eq!(response.tokens_per_sec, Some(1.5)); +} + +/// Non-finite token rates have no JSON representation: they encode as `null` +/// and read back as unknown, rather than failing the whole healthcheck. Pinned +/// deterministically, since the round-trip property only samples them. +#[test] +fn non_finite_token_rates_read_back_as_unknown() { + for rate in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + let response: HealthcheckResponse = serde_json::from_value(serde_json::json!({ + "status": "ready", + "model_loaded": true, + "device": "gpu", + "uptime_sec": 1, + "queue_depth": 0, + })) + .expect("decode"); + let response = HealthcheckResponse { + tokens_per_sec: Some(rate), + ..response + }; + let encoded = serde_json::to_string(&response).expect("a non-finite rate still encodes"); + let decoded: HealthcheckResponse = serde_json::from_str(&encoded).expect("decode"); + assert_eq!( + decoded.tokens_per_sec, None, + "{rate} read back as {:?}", + decoded.tokens_per_sec + ); + } +} + +/// `--gpu` values as a user types them, with the selection each must produce. +#[test] +fn gpu_cli_values_parse_as_documented() { + for (value, expected) in [ + ("auto", Ok(GpuSelection::Auto)), + ("AUTO", Ok(GpuSelection::Auto)), + ("", Ok(GpuSelection::Auto)), + (" ", Ok(GpuSelection::Auto)), + ("0", Ok(GpuSelection::Index(0))), + (" 1 ", Ok(GpuSelection::Index(1))), + ("4294967295", Ok(GpuSelection::Index(u32::MAX))), + ] { + assert_eq!(GpuSelection::parse_cli_value(value), expected, "{value:?}"); + } + let multi = GpuSelection::parse_cli_value("1,2").expect_err("multiple GPUs are rejected"); + assert!( + multi.contains("multiple GPUs is not supported"), + "a list of GPUs must say why it is rejected, got {multi:?}" + ); + for value in ["-1", "4294967296", "gpu0", "1.0", "auto1"] { + assert!( + GpuSelection::parse_cli_value(value).is_err(), + "{value:?} must be rejected" + ); + } +} + +/// Literal request documents as the CLI sends them, and a detect response +/// without the optional runtime fields: each must decode to the values it +/// spells. This catches a renamed or removed field; it cannot catch a lost +/// `#[serde(default)]` on an `Option`, which serde treats as optional anyway. +#[test] +fn request_and_detect_documents_have_their_documented_wire_shape() { + let launch: LaunchRequest = serde_json::from_value(serde_json::json!({ + "service_id": "svc", + "env_id": null, + "runtime_id": null, + "model_ref": "org/model", + "host": "127.0.0.1", + "port": 8080, + "device_policy": "gpu_required", + "endpoint_mode": null, + "gpu_selection": { "index": 1 }, + })) + .expect("a launch request decodes"); + assert_eq!(launch.service_id, "svc"); + assert_eq!(launch.model_ref, "org/model"); + assert_eq!(launch.host, "127.0.0.1"); + assert_eq!(launch.port, 8080); + assert_eq!(launch.device_policy, Some(DevicePolicy::GpuRequired)); + assert_eq!(launch.gpu_selection, Some(GpuSelection::Index(1))); + assert!(launch.engine_recipe.is_none()); + + let logs: LogsRequest = + serde_json::from_value(serde_json::json!({ "service_id": "svc", "tail_lines": 50 })) + .expect("a logs request decodes"); + assert_eq!(logs.service_id, "svc"); + assert_eq!(logs.tail_lines, Some(50)); + + let detect: DetectResponse = serde_json::from_value(serde_json::json!({ + "installed": true, + "env_id": "env", + "python_version": null, + "torch_version": null, + "transformers_version": null, + "available_devices": [{ "kind": "gpu", "available": true }], + "capabilities": { + "cpu": false, + "rocm_gpu": true, + "openai_compatible": true, + "tool_calling": false, + "quantized_models": "none", + "reasoning_parser": false, + }, + "notes": ["n"], + })) + .expect("a detect response without the optional runtime fields decodes"); + assert!(detect.installed); + assert_eq!(detect.env_id.as_deref(), Some("env")); + assert!(detect.runtime_kind.is_none() && detect.managed_env.is_none()); + assert_eq!(detect.available_devices.len(), 1); + assert!(detect.available_devices[0].available); + assert!(detect.capabilities.rocm_gpu && !detect.capabilities.cpu); + assert_eq!(detect.notes, vec!["n".to_owned()]); +} + +/// The error envelope an engine sends, as a literal document: a swapped code +/// and message, or a failure that reads as success, must fail here. +#[test] +fn failure_envelope_has_its_documented_wire_shape() { + assert_eq!( + serde_json::to_value(EngineResponseEnvelope::failure("c", "m")).expect("encode"), + serde_json::json!({ "ok": false, "error": { "code": "c", "message": "m" } }) + ); +} + +/// The plugin binary naming rule, positively: the `rocm-engine-` prefix with +/// an optional `.exe` in either case yields the id, and anything else none. +#[test] +fn plugin_binary_names_yield_their_engine_id() { + for name in [ + "rocm-engine-foo", + "rocm-engine-foo.exe", + "rocm-engine-foo.EXE", + ] { + assert_eq!( + engine_id_from_plugin_binary_name(name).as_deref(), + Some("foo"), + "{name}" + ); + } + for name in ["rocm-engin-foo", "engine-foo", "foo", "rocm-engine-"] { + assert_eq!(engine_id_from_plugin_binary_name(name), None, "{name}"); + } +} + +/// The documented wire name of each method. The `match` is exhaustive, so a +/// new method fails to compile until it is listed here; the variant list in +/// the test below and [`METHOD_WIRE_NAMES`] are maintained by hand and are not +/// compile-enforced. +const fn method_wire_name(method: &EngineMethod) -> &'static str { + match method { + EngineMethod::Detect => "detect", + EngineMethod::Install => "install", + EngineMethod::Capabilities => "capabilities", + EngineMethod::ResolveModel => "resolve_model", + EngineMethod::Launch => "launch", + EngineMethod::Healthcheck => "healthcheck", + EngineMethod::Endpoint => "endpoint", + EngineMethod::Stop => "stop", + EngineMethod::Logs => "logs", + } +} + +/// Every method's wire spelling, written out: the plugin protocol is a +/// contract with separately built engine binaries, so a renamed method must +/// fail here rather than at runtime. (The round-trip properties cannot see a +/// rename, since encoding and decoding change together; the golden documents +/// above pin a few more spellings, not the whole wire format.) +#[test] +fn engine_methods_use_their_documented_wire_names() { + for wire in METHOD_WIRE_NAMES { + let envelope: EngineRequestEnvelope = + serde_json::from_value(serde_json::json!({ "method": wire })) + .unwrap_or_else(|error| panic!("{wire:?} must decode: {error}")); + assert_eq!(method_wire_name(&envelope.method), wire); + } + for method in [ + EngineMethod::Detect, + EngineMethod::Install, + EngineMethod::Capabilities, + EngineMethod::ResolveModel, + EngineMethod::Launch, + EngineMethod::Healthcheck, + EngineMethod::Endpoint, + EngineMethod::Stop, + EngineMethod::Logs, + ] { + let wire = method_wire_name(&method); + assert!( + METHOD_WIRE_NAMES.contains(&wire), + "{wire:?} is missing from METHOD_WIRE_NAMES" + ); + assert_eq!( + serde_json::to_value(&method).expect("encode"), + serde_json::Value::from(wire) + ); + let envelope: EngineRequestEnvelope = + serde_json::from_value(serde_json::json!({ "method": wire })) + .unwrap_or_else(|error| panic!("{wire:?} must decode: {error}")); + assert_eq!(envelope.method, method); + } +} + +// `BTreeMap` keys in a recipe hint come off the wire; duplicates in the JSON +// must resolve deterministically rather than depending on iteration order. +proptest! { + #![proptest_config(ProptestConfig { cases: 256, ..ProptestConfig::default() })] + + #[test] + fn duplicate_parser_setting_keys_resolve_last_wins( + key in "[a-z]{1,6}", + first in "[ -~]{0,8}", + second in "[ -~]{0,8}", + ) { + let json = format!( + r#"{{"contract_version":"0.1.0","engine":"vllm","parser_settings":{{{key}:{first},{key}:{second}}}}}"#, + key = serde_json::to_string(&key).expect("key"), + first = serde_json::to_string(&first).expect("first"), + second = serde_json::to_string(&second).expect("second"), + ); + let decoded: EngineRecipeHint = serde_json::from_str(&json).expect("decode"); + let expected: BTreeMap = BTreeMap::from([(key, second)]); + prop_assert_eq!(decoded.parser_settings, expected); + } +} diff --git a/engines/lemonade/src/archive_cache.rs b/engines/lemonade/src/archive_cache.rs index ad2b7177ce..ed2805eafc 100644 --- a/engines/lemonade/src/archive_cache.rs +++ b/engines/lemonade/src/archive_cache.rs @@ -300,6 +300,124 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// Property: the archive digest check accepts the pinned digest and nothing + /// else. + /// + /// The generator targets near misses rather than random strings, because a + /// random 64-character candidate is rejected by any implementation — + /// correct, prefix-matching, or truncating alike. What separates them is a + /// candidate that shares most of the real digest: one nibble changed, the + /// correct prefix with a wrong tail, a truncated prefix, or the real digest + /// with whitespace or an extra character attached. + /// + /// Note the comparison is byte-exact, so an upper-case spelling of the + /// *correct* digest is also rejected. That is fail-closed (an upper-case + /// pin would break installs, not admit a bad archive) but it differs from + /// `rocm_core`'s download gate, which folds ASCII case. + #[test] + fn archive_digest_check_accepts_only_the_exact_pinned_digest() { + use proptest::prelude::*; + + let dir = scratch_dir("archive-digest-properties"); + let archive = dir.join("archive"); + let content = b"pinned embeddable archive"; + fs::write(&archive, content).unwrap(); + let truth = format!("{:x}", Sha256::digest(content)); + + assert!(verify_sha256(&archive, &truth).is_ok()); + + let mut runner = proptest::test_runner::TestRunner::deterministic(); + let near_miss = { + let truth = truth.clone(); + (0usize..64, 1usize..64, 0u8..7).prop_map(move |(position, keep, kind)| match kind { + 0 => { + let mut out: Vec = truth.chars().collect(); + out[position] = if out[position] == 'a' { 'b' } else { 'a' }; + out.into_iter().collect::() + } + 1 => format!("{}{}", &truth[..keep], "0".repeat(64 - keep)), + 2 => truth[..keep].to_owned(), + 3 => format!("{truth}\n"), + 4 => format!(" {truth} "), + 5 => format!("{truth}0"), + _ => truth.to_ascii_uppercase(), + }) + }; + + runner + .run(&near_miss, |candidate| { + prop_assert_ne!( + &candidate, + &truth, + "the generator must only produce near misses" + ); + prop_assert!( + verify_sha256(&archive, &candidate).is_err(), + "the archive digest check accepted {:?} for an archive whose digest is {:?}", + candidate, + truth + ); + Ok(()) + }) + .unwrap(); + + // An empty pin is the one near miss the generator above cannot produce + // (it keeps at least one character); it must not read as "no check". + assert!( + verify_sha256(&archive, "").is_err(), + "an empty pinned digest accepted the archive" + ); + + fs::remove_dir_all(&dir).ok(); + } + + /// Property: flipping a bit of the archive — at sampled positions, and in + /// the last byte — invalidates it against the pinned digest: the + /// tampered-download case. The archive is longer + /// than any single read buffer, and the intact archive must still be + /// accepted: a check that hashed only a prefix would reject the intact + /// archive and miss flips near the end. + #[test] + fn archive_digest_check_rejects_a_single_bit_of_tampering() { + use proptest::prelude::*; + + let dir = scratch_dir("archive-digest-tamper"); + let archive = dir.join("archive"); + let content: Vec = (0..=255_u8).cycle().take(64 * 1024 + 1).collect(); + let truth = format!("{:x}", Sha256::digest(&content)); + + fs::write(&archive, &content).unwrap(); + assert!( + verify_sha256(&archive, &truth).is_ok(), + "the intact multi-buffer archive must be accepted" + ); + // The last byte is checked explicitly: random positions would almost + // never land on it. + let mut tail_flipped = content.clone(); + *tail_flipped.last_mut().unwrap() ^= 1; + fs::write(&archive, &tail_flipped).unwrap(); + assert!( + verify_sha256(&archive, &truth).is_err(), + "a flip in the last byte was accepted" + ); + + let mut runner = proptest::test_runner::TestRunner::deterministic(); + runner + .run(&(0usize..content.len(), 0u8..8), |(position, bit)| { + let mut tampered = content.clone(); + tampered[position] ^= 1 << bit; + fs::write(&archive, &tampered).unwrap(); + prop_assert!( + verify_sha256(&archive, &truth).is_err(), + "an archive differing by one bit was accepted" + ); + Ok(()) + }) + .unwrap(); + + fs::remove_dir_all(&dir).ok(); + } + #[test] fn repeated_valid_archive_publication_is_idempotent() { let dir = scratch_dir("valid-publication-race"); From 33c9297cff43015ded947fdcd683cbbb847188df Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 1 Oct 2026 13:22:12 +0000 Subject: [PATCH 02/10] test(release): check single-character drifts of the pinned release key The pinned-key check is what guarantees installers trust exactly the key releases are signed with, so a one-character drift during a key rotation must not pass. Runs the real check end to end over a scratch repository root and walks every position of the real published key, in either installer, with a deletion, a duplication, both alphabet neighbours, a case flip and a swap with the next character, rather than sampling random bodies, which any comparison would catch. Also covers a correct copy of the key elsewhere in the same file (before or after the constant), an empty or missing constant against a published key, a blank canonical key (dormant) and a missing installer (error). Pins the reach of the base64 normalization too: whitespace or punctuation inserted into an embedded body is normalized away rather than reported as divergence. Adds a docs/testing.md section listing how to run these suites and the integrity and protocol property tests. Signed-off-by: Roman Inflianskas --- docs/testing.md | 21 ++ xtask/src/verify_pinned_keys.rs | 414 ++++++++++++++++++++++++++++++++ 2 files changed, 435 insertions(+) diff --git a/docs/testing.md b/docs/testing.md index eb9f36502e..579f3594ca 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1835,6 +1835,27 @@ because its runtime key and package specs carry the index's spelling. `generator_reach_report` prints how often each hard case is generated (run it with `-- --nocapture`), and fails if the generator stops reaching them. +Artifact integrity and engine protocol property tests: + +```bash +cargo test -p rocm-core --test digest_gate_properties --test integrity_properties --test model_selector_properties +cargo test -p rocm-engine-protocol --test protocol_properties +cargo test -p rocm-engine-lemonade --lib archive_digest +cargo test -p xtask verify_pinned_keys +``` + +These check the code that decides whether a download, a signature or the +pinned release key is authentic, and the code that decodes engine output. The +generators aim at near misses — a digest one nibble off or sharing a correct +prefix, a signature with one byte changed or leading zeros, a loopback spelling +padded or re-cased, a model id with a variant suffix — because a uniformly +random input is rejected by any implementation, correct or not. The pinned-key +tests run the real check over a scratch repository root and walk every +position of the committed key. `model_selector_properties` also pins, rather +than asserts away, that readiness accepts a bare family word such as `qwen` +reported by an engine for other models of that family, in the catalog and +beyond. + Model recipe engine metadata checks: ```bash diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index 219379b771..41b5c9e65c 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -817,4 +817,418 @@ mod tests { "an unset variable must resolve to None, not to some other variable" ); } + + /// The canonical current release key exactly as committed, so the checks + /// below run against the real trust root rather than a fixture. + fn canonical_release_key_text() -> String { + std::fs::read_to_string(repo_root().join(PINNED_KEYS[0].canonical)) + .expect("the canonical current release key is committed") + } + + /// The canonical PEM's base64 lines, taken from the raw file text by line + /// position (everything between the header and footer lines) rather than + /// through `pem_body`, so a defect in the normalization under test cannot + /// also shape the expected value. + fn raw_body_lines(pem: &str) -> Vec { + let lines: Vec<&str> = pem.lines().collect(); + assert_eq!( + lines.first(), + Some(&PEM_BEGIN), + "canonical key starts with its header" + ); + let footer = lines + .iter() + .position(|line| *line == PEM_END) + .expect("canonical key has a footer"); + lines[1..footer] + .iter() + .map(|line| (*line).to_owned()) + .collect() + } + + fn pem_from_lines(body: &[String]) -> String { + format!("{PEM_BEGIN}\n{}\n{PEM_END}\n", body.join("\n")) + } + + /// The two installer constants the current release key is pinned in, in the + /// shape the real installers embed them. + fn install_sh(current: &str, extra: &str) -> String { + format!( + "#!/bin/sh\n{extra}PINNED_RELEASE_PUBLIC_KEY_CURRENT=\"{current}\"\n\ + PINNED_RELEASE_PUBLIC_KEY_NEXT=\"\"\n" + ) + } + + fn install_ps1(current: &str, extra: &str) -> String { + format!( + "{extra}$PinnedReleasePublicKeyCurrent = @\"\n{current}\"@\n\ + $PinnedReleasePublicKeyNext = \"\"\n" + ) + } + + /// A repository root holding the real canonical current release key and the + /// given installer sources, for running [`check_pinned_keys`] — the real + /// comparison — end to end. The other canonical keys are absent, so their + /// entries are dormant. + fn release_layout(canonical: &str, sh: &str, ps1: &str) -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::create_dir_all(dir.path().join("docs/keys")).expect("create docs/keys"); + std::fs::write(dir.path().join(PINNED_KEYS[0].canonical), canonical) + .expect("write canonical key"); + std::fs::write(dir.path().join("install.sh"), sh).expect("write install.sh"); + std::fs::write(dir.path().join("install.ps1"), ps1).expect("write install.ps1"); + dir + } + + fn diverges(result: &Result>) -> bool { + result + .as_ref() + .is_err_and(|error| error.to_string().contains("have diverged")) + } + + /// Rewrite the installer sources of a [`release_layout`] in place, so a loop + /// over many cases reuses one directory instead of creating one per case. + fn write_sources(root: &Path, sh: &str, ps1: &str) { + std::fs::write(root.join("install.sh"), sh).expect("write install.sh"); + std::fs::write(root.join("install.ps1"), ps1).expect("write install.ps1"); + } + + const BASE64_ALPHABET: &str = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + /// The single-character edits applied at one position of a body line: + /// deletion, duplication, substitution by each neighbour in the base64 + /// alphabet (which swaps `+` and `/`, the two symbols a letters-and-digits + /// normalization would drop; `=` padding has none), for a letter the same + /// letter in the other case, and a swap with the next character when the + /// two differ (which a comparison that ignores character order would miss). + fn single_character_edits(line: &str, column: usize) -> Vec { + let alphabet: Vec = BASE64_ALPHABET.chars().collect(); + let chars: Vec = line.chars().collect(); + let original = chars[column]; + // `=` padding has no alphabet neighbours; deletion and duplication + // still apply to it. (The committed key carries no padding today, so + // this only matters after a rotation to one that does.) + let mut replacements = alphabet + .iter() + .position(|&c| c == original) + .map(|index| { + vec![ + alphabet[(index + 1) % alphabet.len()], + alphabet[(index + alphabet.len() - 1) % alphabet.len()], + ] + }) + .unwrap_or_default(); + if original.is_ascii_alphabetic() { + replacements.push(if original.is_ascii_uppercase() { + original.to_ascii_lowercase() + } else { + original.to_ascii_uppercase() + }); + } + + let mut edits: Vec = replacements + .into_iter() + .map(|replacement| { + let mut edited = chars.clone(); + edited[column] = replacement; + edited.into_iter().collect() + }) + .collect(); + let mut deleted = chars.clone(); + deleted.remove(column); + edits.push(deleted.into_iter().collect()); + let mut duplicated = chars.clone(); + duplicated.insert(column, original); + edits.push(duplicated.into_iter().collect()); + if chars.get(column + 1).is_some_and(|&next| next != original) { + let mut swapped = chars; + swapped.swap(column, column + 1); + edits.push(swapped.into_iter().collect()); + } + edits + } + + /// Every single-character drift of the pinned current release key — at + /// every position of the real published key, in either installer — is + /// reported as divergence by the real check. + /// + /// This is the guarantee the whole check exists for: installers must pin + /// exactly the key CI signs releases with, so a one-character slip during a + /// rotation cannot pass. A random key body is caught by any comparison at + /// all; what separates a correct one from a prefix, truncating or + /// over-normalizing comparison is a body that matches everywhere but one + /// place. So rather than sampling, this walks every position and applies + /// each edit [`single_character_edits`] produces there. Look-alike swaps + /// beyond those (`l`/`I`, `O`/`0`) are not enumerated. + #[test] + fn every_single_character_drift_of_the_pinned_key_is_detected() { + let canonical = canonical_release_key_text(); + let body = raw_body_lines(&canonical); + + // Positive control: the untampered layout passes, so every failure below + // is the drift being detected, not the layout being broken. + let root = release_layout( + &canonical, + &install_sh(&canonical, ""), + &install_ps1(&canonical, ""), + ); + let messages = check_pinned_keys(root.path(), None).expect("the intact layout passes"); + assert!( + messages + .iter() + .any(|m| m.starts_with("release-current: pinned in")), + "the intact layout must actually compare the current key, got {messages:?}" + ); + + let mut expected_checks = 0_usize; + let mut checked = 0_usize; + for (line, text) in body.iter().enumerate() { + for (column, original) in text.chars().enumerate() { + let edits = single_character_edits(text, column); + let swaps = usize::from( + text.chars() + .nth(column + 1) + .is_some_and(|next| next != original), + ); + expected_checks += 2 + * (swaps + + match original { + '=' => 2, + letter if letter.is_ascii_alphabetic() => 5, + _ => 4, + }); + for edit in edits { + let mut tampered_body = body.clone(); + tampered_body[line] = edit; + let tampered = pem_from_lines(&tampered_body); + for (sh, ps1, which) in [ + ( + install_sh(&tampered, ""), + install_ps1(&canonical, ""), + "install.sh", + ), + ( + install_sh(&canonical, ""), + install_ps1(&tampered, ""), + "install.ps1", + ), + ] { + write_sources(root.path(), &sh, &ps1); + let result = check_pinned_keys(root.path(), None); + assert!( + diverges(&result), + "{which}: a key differing at line {line}, column {column} was not \ + reported as divergence: {result:?}" + ); + checked += 1; + } + } + } + } + let body_chars: usize = body.iter().map(|line| line.chars().count()).sum(); + assert!( + body_chars > 300, + "the walk covered only {body_chars} body characters" + ); + assert_eq!(checked, expected_checks, "every position gets every edit"); + } + + /// A published canonical key whose pinned constant is empty, or missing + /// from the source altogether, is divergence — not a match and not a + /// dormant skip. Each case leaves exactly one entry wrong, so the error + /// must name that entry. + #[test] + fn an_empty_or_missing_constant_against_a_published_key_diverges() { + let canonical = canonical_release_key_text(); + let root = release_layout( + &canonical, + &install_sh(&canonical, ""), + &install_ps1(&canonical, ""), + ); + let next_canonical = root.path().join(PINNED_KEYS[1].canonical); + + let cases = [ + ( + "current empty in install.sh", + install_sh("", ""), + install_ps1(&canonical, ""), + false, + "release-current", + ), + ( + "current empty in install.ps1", + install_sh(&canonical, ""), + install_ps1("", ""), + false, + "release-current", + ), + ( + "current constant missing from install.sh", + "#!/bin/sh\nPINNED_RELEASE_PUBLIC_KEY_NEXT=\"\"\n".to_owned(), + install_ps1(&canonical, ""), + false, + "release-current", + ), + // The rotation case: the next key's PEM is published (the same key + // stands in for it) while the installers still carry `NEXT=""`. + ( + "next published but empty in the installers", + install_sh(&canonical, ""), + install_ps1(&canonical, ""), + true, + "release-next", + ), + ]; + for (label, sh, ps1, publish_next, entry) in cases { + if publish_next { + std::fs::write(&next_canonical, &canonical).expect("write canonical next key"); + } else { + let _ = std::fs::remove_file(&next_canonical); + } + write_sources(root.path(), &sh, &ps1); + let result = check_pinned_keys(root.path(), None); + assert!( + diverges(&result), + "{label}: expected divergence, got {result:?}" + ); + let message = result.expect_err("diverged").to_string(); + assert!( + message.starts_with(&format!("{entry}:")), + "{label}: the divergence must be reported for {entry}, got {message}" + ); + } + } + + /// An empty canonical file is a key not yet published (dormant), while a + /// pinned source file that does not exist is an error naming the file — + /// neither may read as a match. + #[test] + fn a_blank_canonical_key_is_dormant_and_a_missing_source_is_an_error() { + let canonical = canonical_release_key_text(); + let root = release_layout( + " \n", + &install_sh(&canonical, ""), + &install_ps1(&canonical, ""), + ); + let messages = + check_pinned_keys(root.path(), None).expect("a blank canonical key is dormant"); + assert!( + messages + .iter() + .any(|m| m == "release-current: no canonical key yet — skipped (dormant)"), + "a blank canonical key must be reported dormant, got {messages:?}" + ); + + std::fs::write(root.path().join(PINNED_KEYS[0].canonical), &canonical) + .expect("write canonical key"); + std::fs::remove_file(root.path().join("install.ps1")).expect("remove install.ps1"); + let error = check_pinned_keys(root.path(), None) + .expect_err("a missing pinned source must be an error"); + assert_eq!( + error.to_string(), + "release-current: source file is missing: install.ps1" + ); + } + + /// A correct copy of the canonical key elsewhere in the same file — in a + /// comment, in another constant, in a stale copy, before or after the + /// constant — never makes a wrong `CURRENT` constant pass. + /// + /// The decoys are placed *before* the constant as well as after it, because + /// a check that reads the first key in the file would only be caught by a + /// decoy that comes first. + #[test] + fn a_correct_key_elsewhere_in_the_file_never_rescues_a_wrong_constant() { + let canonical = canonical_release_key_text(); + let mut body = raw_body_lines(&canonical); + // Flip the case of the first letter on the second body line, a + // one-character drift that stays inside the base64 alphabet. + let flipped: String = { + let mut chars: Vec = body[1].chars().collect(); + let at = chars + .iter() + .position(char::is_ascii_alphabetic) + .expect("a letter to flip"); + chars[at] = if chars[at].is_ascii_uppercase() { + chars[at].to_ascii_lowercase() + } else { + chars[at].to_ascii_uppercase() + }; + chars.into_iter().collect() + }; + body[1] = flipped; + let wrong = pem_from_lines(&body); + assert_fixture_differs(&wrong, &canonical); + + let sh_decoys = [ + format!("# previous key:\n# {}\n", canonical.replace('\n', "\n# ")), + format!("PREVIOUS_RELEASE_PUBLIC_KEY=\"{canonical}\"\n"), + format!("STALE_COPY=\"{canonical}\"\n"), + ]; + let ps1_decoys = [ + format!("# previous key:\n# {}\n", canonical.replace('\n', "\n# ")), + format!("$PreviousReleasePublicKey = @\"\n{canonical}\"@\n"), + format!("$StaleCopy = \"{canonical}\"\n"), + ]; + + for decoy in &sh_decoys { + for sh in [ + install_sh(&wrong, decoy), + format!("{}{decoy}", install_sh(&wrong, "")), + ] { + let root = release_layout(&canonical, &sh, &install_ps1(&canonical, "")); + let result = check_pinned_keys(root.path(), None); + assert!( + diverges(&result), + "install.sh: a wrong CURRENT passed because the canonical key appears \ + elsewhere ({decoy:?}): {result:?}" + ); + } + } + for decoy in &ps1_decoys { + for ps1 in [ + install_ps1(&wrong, decoy), + format!("{}{decoy}", install_ps1(&wrong, "")), + ] { + let root = release_layout(&canonical, &install_sh(&canonical, ""), &ps1); + let result = check_pinned_keys(root.path(), None); + assert!( + diverges(&result), + "install.ps1: a wrong CURRENT passed because the canonical key appears \ + elsewhere ({decoy:?}): {result:?}" + ); + } + } + } + + fn assert_fixture_differs(left: &str, right: &str) { + assert_ne!(left, right, "the fixture must actually differ"); + } + + /// Characterization: the comparison reduces each key to its base64 + /// alphabet, so characters *outside* that alphabet inserted into an + /// embedded key's body — whitespace or punctuation — are normalized away + /// and do not register as divergence. + /// + /// Whether such a body is harmless rests on the consumers' PEM readers + /// (OpenSSL in the installers, `pem-rfc7468` in the binary) rejecting or + /// ignoring it at load time; this check does not decide that. Pinned so the + /// normalization's reach is visible rather than implicit. + #[test] + fn non_base64_noise_in_an_embedded_key_is_normalized_away() { + let canonical = canonical_release_key_text(); + for noise in [" ", "!", "."] { + let noisy = canonical.replacen("\nMII", &format!("\n{noise}MII"), 1); + assert_fixture_differs(&noisy, &canonical); + let root = release_layout( + &canonical, + &install_sh(&noisy, ""), + &install_ps1(&noisy, ""), + ); + check_pinned_keys(root.path(), None).unwrap_or_else(|error| { + panic!("{noise:?} inside the body is expected to be normalized away: {error:#}") + }); + } + } } From b8a6bcc01d7bad7987b4a1dcc21b6c19d577797c Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 5 Nov 2026 16:29:47 +0200 Subject: [PATCH 03/10] test(integrity): pin alias/basename readiness matches, tighten checks The catalog now carries aliases that spell another catalog model's basename: `qwen3-32b` is an alias of Qwen/Qwen3-32B-FP8 and the basename of Qwen/Qwen3-32B. Readiness accepts a basename for its model, so an engine serving the FP8 recipe under that alias reads as ready for the other one, and the exhaustive "a different catalog model is never readiness" property failed on that. The behaviour is unchanged here; it is excluded from that property and pinned in its own test against the catalog's current list of collisions, so a catalog change that adds one fails rather than being silently skipped. Rejections now have to be the gate's own: the digest tests require the `SHA-256 mismatch` error and that the request reached the server, so a transport failure cannot pass as a rejected digest. The loopback server serves any number of requests and cannot hang when no client connects, which also lets the pairwise readiness test use one server per served name instead of one per request. The one-nibble and prefix near misses are walked over every position instead of sampled. Every property's case count can now be raised by PROPTEST_CASES but not lowered. The Lemonade archive-digest properties took their count from it unchecked, so PROPTEST_CASES=0 passed them without running a case. The removed selector-uniqueness test duplicated the catalog schema's own check, and two protocol properties drop assertions another one in the same test already implied. Signed-off-by: Roman Inflianskas --- crates/rocm-core/tests/common/loopback.rs | 108 ++++++ .../rocm-core/tests/common/proptest_cases.rs | 49 +++ .../rocm-core/tests/digest_gate_properties.rs | 155 ++++++--- .../rocm-core/tests/integrity_properties.rs | 14 +- .../tests/model_selector_properties.rs | 320 ++++++++++-------- .../tests/protocol_properties.rs | 50 ++- docs/testing.md | 20 +- engines/lemonade/src/archive_cache.rs | 34 +- engines/lemonade/src/test_support.rs | 4 + 9 files changed, 535 insertions(+), 219 deletions(-) create mode 100644 crates/rocm-core/tests/common/loopback.rs create mode 100644 crates/rocm-core/tests/common/proptest_cases.rs diff --git a/crates/rocm-core/tests/common/loopback.rs b/crates/rocm-core/tests/common/loopback.rs new file mode 100644 index 0000000000..40c05aa79f --- /dev/null +++ b/crates/rocm-core/tests/common/loopback.rs @@ -0,0 +1,108 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A loopback HTTP server for the integration tests that drive real network +//! code: it answers every request with one fixed `200 OK` until it is dropped. + +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::thread; +use std::time::Duration; + +/// Serves one fixed response on an ephemeral loopback port, to any number of +/// requests, until dropped. +/// +/// Dropping it never hangs: it wakes the accept loop with a connection of its +/// own before joining, so a client that never connected cannot leave the +/// server thread blocked in `accept`. +pub struct LoopbackServer { + port: u16, + stopping: Arc, + served: Arc, + handle: Option>, +} + +impl LoopbackServer { + pub fn start(content_type: Option<&str>, body: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + let port = listener.local_addr().expect("local addr").port(); + let stopping = Arc::new(AtomicBool::new(false)); + let served = Arc::new(AtomicUsize::new(0)); + let content_type = content_type + .map(|value| format!("Content-Type: {value}\r\n")) + .unwrap_or_default(); + let handle = thread::spawn({ + let stopping = Arc::clone(&stopping); + let served = Arc::clone(&served); + move || { + for stream in listener.incoming() { + if stopping.load(Ordering::SeqCst) { + break; + } + let Ok(mut stream) = stream else { continue }; + // A client that connects and never sends cannot hold the + // accept loop, and with it `Drop`, forever. + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + read_request_head(&mut stream); + // Counted before the response is written, so a client that + // has its answer always sees its request counted. + served.fetch_add(1, Ordering::SeqCst); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\n{content_type}Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(&body); + let _ = stream.flush(); + } + } + }); + Self { + port, + stopping, + served, + handle: Some(handle), + } + } + + pub fn url(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } + + /// Requests answered so far. A test that expects a verdict about the + /// response checks this is non-zero, so a request that never reached the + /// server cannot pass as one the code under test rejected. + pub fn requests_served(&self) -> usize { + self.served.load(Ordering::SeqCst) + } +} + +impl Drop for LoopbackServer { + fn drop(&mut self) { + self.stopping.store(true, Ordering::SeqCst); + // Wake the accept loop so it sees the flag. If even that connection + // fails, leave the thread detached rather than block on it. + let woken = TcpStream::connect(("127.0.0.1", self.port)).is_ok(); + if let Some(handle) = self.handle.take() + && woken + { + let _ = handle.join(); + } + } +} + +/// Read the request line and headers, so the response is not written into a +/// request still being sent. +fn read_request_head(stream: &mut TcpStream) { + let mut head = Vec::new(); + let mut chunk = [0_u8; 1024]; + while !head.windows(4).any(|window| window == b"\r\n\r\n") && head.len() < 64 * 1024 { + match stream.read(&mut chunk) { + Ok(0) | Err(_) => break, + Ok(read) => head.extend_from_slice(&chunk[..read]), + } + } +} diff --git a/crates/rocm-core/tests/common/proptest_cases.rs b/crates/rocm-core/tests/common/proptest_cases.rs new file mode 100644 index 0000000000..177fdfa460 --- /dev/null +++ b/crates/rocm-core/tests/common/proptest_cases.rs @@ -0,0 +1,49 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The case-count rule for property tests that set their own count, shared by +//! the `rocm-core` and `rocm-engine-protocol` integration tests and the +//! Lemonade engine's unit tests, which include this file by `#[path]`. +//! +//! It is the same rule as `rocm-core`'s crate-internal +//! `test_support::proptest_cases`, which integration tests and other crates +//! cannot reach. + +/// Case count for a property test: `PROPTEST_CASES` (proptest's own knob, +/// which an explicit count would otherwise override) when it is set and above +/// `default`, otherwise `default`. +/// +/// Never below `default`: each count is chosen so its property reaches the +/// cases it targets, and `PROPTEST_CASES=0` would otherwise pass every +/// property without running one. A value that is ignored, because it is lower +/// or does not parse, is reported on stderr. +pub fn proptest_cases(default: u32) -> u32 { + let requested = std::env::var("PROPTEST_CASES").ok(); + let cases = raised_case_count(requested.as_deref(), default); + if let Some(requested) = requested + && requested.parse::().ok() != Some(cases) + { + eprintln!( + "PROPTEST_CASES={requested} ignored: this property runs at least {default} cases" + ); + } + cases +} + +/// The pure half of [`proptest_cases`]: `requested` raises `default` and never +/// lowers it. +pub fn raised_case_count(requested: Option<&str>, default: u32) -> u32 { + requested + .and_then(|value| value.parse().ok()) + .map_or(default, |requested: u32| requested.max(default)) +} + +#[test] +fn proptest_cases_can_raise_the_default_but_never_lower_it() { + assert_eq!(raised_case_count(None, 24), 24); + assert_eq!(raised_case_count(Some("1000"), 24), 1000); + assert_eq!(raised_case_count(Some("0"), 24), 24); + assert_eq!(raised_case_count(Some("8"), 24), 24); + assert_eq!(raised_case_count(Some("lots"), 24), 24); +} diff --git a/crates/rocm-core/tests/digest_gate_properties.rs b/crates/rocm-core/tests/digest_gate_properties.rs index bb32bf17b4..93a0a2f22b 100644 --- a/crates/rocm-core/tests/digest_gate_properties.rs +++ b/crates/rocm-core/tests/digest_gate_properties.rs @@ -12,36 +12,21 @@ //! from a prefix comparison, a truncating comparison, or a trimming one is a //! candidate that shares most of the real digest. -use std::io::{Read, Write}; -use std::net::TcpListener; +#[path = "common/loopback.rs"] +mod loopback; +#[path = "common/proptest_cases.rs"] +mod proptest_cases; + use std::path::PathBuf; use std::thread; use std::time::Duration; +use loopback::LoopbackServer; use proptest::prelude::*; +use proptest_cases::proptest_cases; use rocm_core::{DownloadRequest, download_file_streaming}; use sha2::{Digest, Sha256}; -/// Serve `body` once over HTTP on an ephemeral loopback port. -fn serve_once(body: Vec) -> (u16, thread::JoinHandle<()>) { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); - let port = listener.local_addr().expect("local addr").port(); - let handle = thread::spawn(move || { - if let Ok((mut stream, _)) = listener.accept() { - let mut scratch = [0_u8; 2048]; - let _ = stream.read(&mut scratch); - let _ = write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.len() - ); - let _ = stream.write_all(&body); - let _ = stream.flush(); - } - }); - (port, handle) -} - fn scratch_dir(tag: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!( "rocm-core-digest-gate-{tag}-{}-{:?}", @@ -53,26 +38,39 @@ fn scratch_dir(tag: &str) -> PathBuf { dir } -/// Run one download against a one-shot server, returning whether the gate +/// Run one download against a loopback server, returning whether the gate /// accepted the body. fn download_accepts(body: &[u8], expected_sha256: Option<&str>, tag: &str) -> bool { - download_outcome(body, expected_sha256, tag).0 + download_outcome(body, expected_sha256, tag).0.is_ok() } -/// Like [`download_accepts`], also returning every file the download left in -/// its otherwise empty scratch directory — the destination and any partial -/// file, whatever it is named. -fn download_outcome(body: &[u8], expected_sha256: Option<&str>, tag: &str) -> (bool, Vec) { +/// Run one download against a loopback server, returning the download's +/// result and every file it left in its otherwise empty scratch directory — +/// the destination and any partial file, whatever it is named. +/// +/// Panics if the request never reached the server: a download that failed +/// before the body arrived proves nothing about the digest gate. +fn download_outcome( + body: &[u8], + expected_sha256: Option<&str>, + tag: &str, +) -> (Result<(), String>, Vec) { let dir = scratch_dir(tag); let destination = dir.join("artifact.bin"); - let (port, server) = serve_once(body.to_vec()); - let url = format!("http://127.0.0.1:{port}/artifact.bin"); + let server = LoopbackServer::start(None, body.to_vec()); + let url = format!("{}/artifact.bin", server.url()); let request = DownloadRequest { expected_sha256, ..DownloadRequest::new(&url, &destination, Duration::from_secs(10)) }; - let accepted = download_file_streaming(&request).is_ok(); - let _ = server.join(); + let result = download_file_streaming(&request) + .map(drop) + .map_err(|error| format!("{error:#}")); + assert!( + server.requests_served() > 0, + "the download never reached the test server: {result:?}" + ); + drop(server); let mut left_behind: Vec = std::fs::read_dir(&dir) .expect("read scratch dir") .map(|entry| { @@ -85,11 +83,19 @@ fn download_outcome(body: &[u8], expected_sha256: Option<&str>, tag: &str) -> (b .collect(); left_behind.sort(); let _ = std::fs::remove_dir_all(&dir); - (accepted, left_behind) + (result, left_behind) +} + +/// Whether `result` is the digest gate's rejection, rather than success or +/// some other failure. +fn is_digest_rejection(result: &Result<(), String>) -> bool { + result + .as_ref() + .is_err_and(|error| error.contains("SHA-256 mismatch")) } /// The chunk size the download streams through, copied from the private -/// `DOWNLOAD_CHUNK_BYTES` in `net.rs`. The multi-chunk body below spans three +/// `DOWNLOAD_CHUNK_BYTES` in `download.rs`. The multi-chunk body below spans three /// of these plus a byte, so it still crosses a chunk boundary if that constant /// grows a little, and regardless of how short the socket reads come back. const CHUNK_BYTES: usize = 64 * 1024; @@ -161,16 +167,57 @@ fn apply_near_miss(digest: &str, miss: NearMiss, position: usize, keep: usize) - } } +/// The positional near misses — one nibble changed, a correct prefix with a +/// wrong tail, a truncated prefix — at every position and every prefix length, +/// rather than at positions drawn at random: a regression that misses only one +/// position (a comparison that skips the last nibble, say) is caught on every +/// run, not on the runs that happen to draw it. +#[test] +fn positional_near_misses_are_rejected_at_every_position() { + let body = b"artifact body for the positional near misses".to_vec(); + let truth = digest_of(&body); + let mut not_rejected = Vec::new(); + for miss in [ + NearMiss::OneNibble, + NearMiss::WrongSuffix, + NearMiss::TruncatedPrefix, + ] { + for at in 0..truth.len() { + // `apply_near_miss` reads `position` for a changed nibble and + // `keep` (as `1 + keep % 63`) for a prefix length; `at` walks both. + let candidate = apply_near_miss(&truth, miss, at, at); + assert!( + !candidate.eq_ignore_ascii_case(&truth), + "near-miss kind {miss:?} at {at} reproduced the true digest" + ); + let (result, _) = download_outcome(&body, Some(&candidate), "positional"); + if !is_digest_rejection(&result) { + not_rejected.push(format!("{miss:?} at {at}: {candidate:?} gave {result:?}")); + } + } + } + assert!( + not_rejected.is_empty(), + "the digest gate did not reject these near misses of {truth:?}:\n{}", + not_rejected.join("\n") + ); +} + proptest! { - #![proptest_config(ProptestConfig { cases: 24, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(24), + ..ProptestConfig::default() + })] - /// A near-miss digest must never be accepted. The one deliberate exception - /// is pure ASCII case folding, which the gate implements as equivalent and - /// which is covered separately below. + /// A near-miss digest must never be accepted, whatever the body. The one + /// deliberate exception is pure ASCII case folding, which the gate + /// implements as equivalent and which is covered separately below. /// /// Every kind of near miss is tried for every generated case, rather than /// one kind drawn per case: a regression that admits just one kind would - /// otherwise go unnoticed in any run that happened not to draw it. + /// otherwise go unnoticed in any run that happened not to draw it. The + /// positional kinds are also walked exhaustively by + /// [`positional_near_misses_are_rejected_at_every_position`]. #[test] fn near_miss_digests_are_rejected( body in prop::collection::vec(any::(), 1..512), @@ -184,17 +231,21 @@ proptest! { !candidate.eq_ignore_ascii_case(&truth), "near-miss kind {miss:?} reproduced the true digest" ); + let (result, _) = download_outcome(&body, Some(&candidate), "near-miss"); prop_assert!( - !download_accepts(&body, Some(&candidate), "near-miss"), - "the digest gate accepted {candidate:?} for an artifact whose digest is {truth:?} \ - (near-miss kind {miss:?})" + is_digest_rejection(&result), + "the digest gate did not reject {candidate:?} for an artifact whose digest is \ + {truth:?} (near-miss kind {miss:?}): {result:?}" ); } } } proptest! { - #![proptest_config(ProptestConfig { cases: 12, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(12), + ..ProptestConfig::default() + })] /// The true digest is accepted, in either ASCII case, and in mixed case — /// case-insensitivity must hold in both directions, not just for a fully @@ -227,10 +278,10 @@ proptest! { let mut tampered = body; let position = index.index(tampered.len()); tampered[position] ^= 1 << bit; - let (accepted, left_behind) = download_outcome(&tampered, Some(&truth), "tampered"); + let (result, left_behind) = download_outcome(&tampered, Some(&truth), "tampered"); prop_assert!( - !accepted, - "a body differing by one bit was accepted against the original digest" + is_digest_rejection(&result), + "a body differing by one bit was not rejected against the original digest: {result:?}" ); prop_assert!( left_behind.is_empty(), @@ -254,8 +305,11 @@ fn tampering_past_the_first_chunk_is_rejected() { ); let mut tampered = body; *tampered.last_mut().expect("non-empty body") ^= 1; - let (accepted, left_behind) = download_outcome(&tampered, Some(&truth), "multi-chunk"); - assert!(!accepted, "a change in the last byte was not detected"); + let (result, left_behind) = download_outcome(&tampered, Some(&truth), "multi-chunk"); + assert!( + is_digest_rejection(&result), + "a change in the last byte was not detected: {result:?}" + ); assert!( left_behind.is_empty(), "a rejected download left {left_behind:?} behind" @@ -263,7 +317,10 @@ fn tampering_past_the_first_chunk_is_rejected() { } proptest! { - #![proptest_config(ProptestConfig { cases: 8, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(8), + ..ProptestConfig::default() + })] /// Without an expected digest the download succeeds — this is the gate's /// documented fail-open mode, pinned here so it cannot be mistaken for diff --git a/crates/rocm-core/tests/integrity_properties.rs b/crates/rocm-core/tests/integrity_properties.rs index aade990e53..eb782735ab 100644 --- a/crates/rocm-core/tests/integrity_properties.rs +++ b/crates/rocm-core/tests/integrity_properties.rs @@ -11,9 +11,13 @@ //! random generator essentially never produces an input that reaches the //! interesting branches of an RSA verifier. +#[path = "common/proptest_cases.rs"] +mod proptest_cases; + use std::sync::OnceLock; use proptest::prelude::*; +use proptest_cases::proptest_cases; use rocm_core::{ generate_rsa_signing_keypair, sign_rsa_pkcs1_sha256_signature, verify_rsa_pkcs1_sha256_signature, @@ -52,7 +56,10 @@ fn payload_strategy() -> impl Strategy> { } proptest! { - #![proptest_config(ProptestConfig { cases: 64, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(64), + ..ProptestConfig::default() + })] /// Baseline: a genuine signature over the exact payload verifies. #[test] @@ -169,7 +176,10 @@ fn mangle_pem(pem: &str, style: u8) -> String { } proptest! { - #![proptest_config(ProptestConfig { cases: 48, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(48), + ..ProptestConfig::default() + })] /// Tolerated mangling must not change *which* key is trusted. #[test] diff --git a/crates/rocm-core/tests/model_selector_properties.rs b/crates/rocm-core/tests/model_selector_properties.rs index 522599b0f7..75b56089eb 100644 --- a/crates/rocm-core/tests/model_selector_properties.rs +++ b/crates/rocm-core/tests/model_selector_properties.rs @@ -13,13 +13,16 @@ //! never panic and must not report a *different* model as the one the user //! asked for. -use std::collections::BTreeMap; -use std::io::{Read, Write}; -use std::net::TcpListener; -use std::thread; +#[path = "common/loopback.rs"] +mod loopback; +#[path = "common/proptest_cases.rs"] +mod proptest_cases; + use std::time::Duration; +use loopback::LoopbackServer; use proptest::prelude::*; +use proptest_cases::proptest_cases; use rocm_core::{ModelRecipeRecord, builtin_model_recipes, openai_models_endpoint_has_model}; fn catalog() -> &'static [ModelRecipeRecord] { @@ -43,34 +46,37 @@ fn advertised_selectors() -> Vec<(String, String)> { selectors } -/// Serve one HTTP response carrying `body` on an ephemeral loopback port. -fn serve_once(body: String) -> (u16, thread::JoinHandle<()>) { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); - let port = listener.local_addr().expect("local addr").port(); - let handle = thread::spawn(move || { - if let Ok((mut stream, _)) = listener.accept() { - let mut scratch = [0_u8; 4096]; - let _ = stream.read(&mut scratch); - let _ = write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.len() - ); - let _ = stream.write_all(body.as_bytes()); - let _ = stream.flush(); - } - }); - (port, handle) +/// A `/v1/models` document listing exactly `ids`. +fn models_document(ids: &[&str]) -> String { + serde_json::json!({ + "object": "list", + "data": ids + .iter() + .map(|id| serde_json::json!({ "id": id, "object": "model" })) + .collect::>(), + }) + .to_string() +} + +/// Ask the readiness check what it makes of the document `server` serves when +/// expecting `expected`. Panics if the request never reached the server, so a +/// transport failure cannot pass as a "not ready" verdict. +fn readiness_from(server: &LoopbackServer, expected: Option<&str>) -> Result { + let before = server.requests_served(); + let outcome = + openai_models_endpoint_has_model(&server.url(), expected, None, Duration::from_secs(10)) + .map_err(|error| error.to_string()); + assert!( + server.requests_served() > before, + "the readiness check never reached the test server: {outcome:?}" + ); + outcome } /// Ask the readiness check what it makes of `body` when expecting `expected`. fn endpoint_reports(body: String, expected: Option<&str>) -> Result { - let (port, server) = serve_once(body); - let url = format!("http://127.0.0.1:{port}"); - let outcome = openai_models_endpoint_has_model(&url, expected, None, Duration::from_secs(10)) - .map_err(|error| error.to_string()); - let _ = server.join(); - outcome + let server = LoopbackServer::start(Some("application/json"), body.into_bytes()); + readiness_from(&server, expected) } #[test] @@ -81,30 +87,11 @@ fn the_shipped_catalog_is_not_empty() { ); } -/// No selector the catalog advertises may resolve to two different recipes. -/// `resolve_model_recipe` returns the first match, so an overlap means the -/// winner depends on JSON ordering rather than on the user's intent. -#[test] -fn no_advertised_selector_matches_two_recipes() { - let mut ambiguous: BTreeMap> = BTreeMap::new(); - for (selector, _) in advertised_selectors() { - let matches: Vec<&str> = catalog() - .iter() - .filter(|recipe| recipe.matches_ref(&selector)) - .map(|recipe| recipe.canonical_model_id.as_str()) - .collect(); - if matches.len() > 1 { - ambiguous.insert(selector, matches); - } - } - assert!( - ambiguous.is_empty(), - "these selectors resolve to more than one recipe, so the winner is catalog order: \ - {ambiguous:#?}" - ); -} - /// Every advertised selector must resolve to the recipe that advertises it. +/// +/// That no selector is advertised twice is the catalog schema's own check +/// (`ModelRecipeIndexDocument::validate`, which normalizes refs the same way +/// `matches_ref` does), run on the shipped catalog by its unit tests. #[test] fn every_advertised_selector_resolves_to_its_own_recipe() { for (selector, owner) in advertised_selectors() { @@ -122,7 +109,10 @@ fn every_advertised_selector_resolves_to_its_own_recipe() { } proptest! { - #![proptest_config(ProptestConfig { cases: 512, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(512), + ..ProptestConfig::default() + })] /// Selector matching ignores surrounding whitespace and ASCII case: any /// re-casing of an advertised selector, padded with spaces, tabs or @@ -212,19 +202,18 @@ proptest! { } } -/// `/v1/models` bodies shaped the way a real engine shapes them, plus the -/// degenerate shapes a broken or hostile one could send. +/// `/v1/models` bodies that name no model matching any of the expectations +/// below, shaped the way a real engine shapes them, plus the degenerate shapes +/// a broken or hostile one could send. +/// +/// The listed ids carry no letter. Every expected name below does, and every +/// rule readiness matches by — equality, basename, catalog alias, family word — +/// needs a letter in common, so none of these documents may read as ready. fn models_body_strategy() -> impl Strategy { prop_oneof![ - 3 => prop::collection::vec("[ -~]{0,24}", 0..4).prop_map(|ids| { - serde_json::json!({ - "object": "list", - "data": ids - .into_iter() - .map(|id| serde_json::json!({ "id": id, "object": "model" })) - .collect::>(), - }) - .to_string() + 3 => prop::collection::vec("[0-9 ._:#/@-]{0,24}", 0..4).prop_map(|ids| { + let ids: Vec<&str> = ids.iter().map(String::as_str).collect(); + models_document(&ids) }), 1 => Just("{}".to_owned()), 1 => Just(r#"{"data": []}"#.to_owned()), @@ -240,12 +229,17 @@ fn models_body_strategy() -> impl Strategy { } proptest! { - #![proptest_config(ProptestConfig { cases: 48, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(48), + ..ProptestConfig::default() + })] - /// A malformed or hostile `/v1/models` document must produce a clean - /// `false` or a clean error — never a panic, and never a spurious "ready". + /// A `/v1/models` document that names no matching model — malformed, + /// hostile, or listing only unrelated ids — must produce a clean `false` or + /// a clean error: never a panic, and never "ready". A check that reported + /// ready for any non-empty listing, ignoring the expected model, fails here. #[test] - fn a_models_document_never_reports_ready_without_naming_a_model( + fn a_models_document_naming_no_matching_model_never_reports_ready( body in models_body_strategy(), expected in prop::sample::select(vec![ "Qwen/Qwen3-4B-Instruct-2507", @@ -253,27 +247,14 @@ proptest! { "nonexistent/model", ]), ) { - // The call must return, not unwind. Any `Ok(true)` has to be justified - // by the document actually naming a model that matches. - if endpoint_reports(body.clone(), Some(expected)) == Ok(true) { - let value: serde_json::Value = serde_json::from_str(body.trim()) - .expect("a ready verdict requires a parseable document"); - let names_a_model = value - .get("data") - .and_then(serde_json::Value::as_array) - .into_iter() - .flatten() - .any(|item| { - ["id", "model", "name"] - .into_iter() - .filter_map(|field| item.get(field).and_then(serde_json::Value::as_str)) - .any(|value| !value.trim().is_empty()) - }); - prop_assert!( - names_a_model, - "reported ready for {expected:?} from a document naming no model at all: {body}" - ); - } + let verdict = endpoint_reports(body.clone(), Some(expected)); + prop_assert_ne!( + verdict, + Ok(true), + "reported ready for {:?} from a document naming no matching model: {}", + expected, + body + ); } /// Positive control for the property above: a document that does list the @@ -303,14 +284,8 @@ proptest! { let mut ids = others; let position = at.index(ids.len() + 1); ids.insert(position, listed.clone()); - let body = serde_json::json!({ - "object": "list", - "data": ids - .iter() - .map(|id| serde_json::json!({ "id": id, "object": "model" })) - .collect::>(), - }) - .to_string(); + let ids: Vec<&str> = ids.iter().map(String::as_str).collect(); + let body = models_document(&ids); prop_assert_eq!( endpoint_reports(body, Some(expected)), Ok(true), @@ -365,33 +340,42 @@ fn is_bare_family_word(name: &str) -> bool { /// and a family match from one that has lost its guards or matches inside a /// word rather than at its start. /// -/// Bare family words are excluded here and pinned by -/// [`a_bare_family_word_satisfies_readiness_for_the_whole_family`], which -/// documents the one shape that *does* cross-match today. +/// Two shapes that *do* cross-match today are excluded here and pinned +/// separately, so they stay visible: bare family words, by +/// [`a_bare_family_word_satisfies_readiness_for_the_whole_family`], and one +/// recipe's alias spelling another recipe's basename, by +/// [`an_alias_spelling_another_models_basename_reads_as_ready_for_it`]. +/// +/// The request count grows with the square of the catalog (every served name +/// against every wanted model), so each served name gets one loopback server +/// that answers all of its requests, rather than a server per request. #[test] fn listing_a_different_catalog_model_is_never_readiness() { let recipes = catalog(); assert!(recipes.len() > 1, "need at least two recipes to confuse"); let mut false_positives = Vec::new(); - for wanted in recipes { - for served in recipes { - if served.canonical_model_id == wanted.canonical_model_id { + for served in recipes { + let served_names = std::iter::once(&served.canonical_model_id).chain(&served.aliases); + for served_name in served_names { + if is_bare_family_word(served_name) { continue; } - let served_names = std::iter::once(&served.canonical_model_id).chain(&served.aliases); - for served_name in served_names { - // Skip a name the *wanted* recipe also answers to: that is a - // genuine shared alias, not a confusion. - if wanted.matches_ref(served_name) || is_bare_family_word(served_name) { + let server = LoopbackServer::start( + Some("application/json"), + models_document(&[served_name.as_str()]).into_bytes(), + ); + for wanted in recipes { + // Skip the served recipe itself, a name the *wanted* recipe + // also answers to (a genuine shared alias, not a confusion), + // and the pinned alias/basename collision. + if wanted.canonical_model_id == served.canonical_model_id + || wanted.matches_ref(served_name) + || spells_the_basename_of(served_name, wanted) + { continue; } - let body = serde_json::json!({ - "object": "list", - "data": [{ "id": served_name, "object": "model" }], - }) - .to_string(); - if endpoint_reports(body, Some(&wanted.canonical_model_id)) == Ok(true) { + if readiness_from(&server, Some(&wanted.canonical_model_id)) == Ok(true) { false_positives.push(format!( "wanted {:?} but an endpoint serving only {:?} reported ready", wanted.canonical_model_id, served_name @@ -399,16 +383,14 @@ fn listing_a_different_catalog_model_is_never_readiness() { } } } + } + for wanted in recipes { for served_name in near_miss_names(&wanted.canonical_model_id) { if wanted.matches_ref(&served_name) { continue; } - let body = serde_json::json!({ - "object": "list", - "data": [{ "id": served_name, "object": "model" }], - }) - .to_string(); + let body = models_document(&[served_name.as_str()]); if endpoint_reports(body, Some(&wanted.canonical_model_id)) == Ok(true) { false_positives.push(format!( "wanted {:?} but an endpoint serving only the near miss {:?} reported ready", @@ -425,6 +407,80 @@ fn listing_a_different_catalog_model_is_never_readiness() { ); } +/// Whether `name`, as an engine would report it, is exactly the basename of +/// `recipe`'s canonical id (up to ASCII case) — the part after the last `/`, +/// which readiness accepts on its own because engines commonly report a model +/// by its basename. +fn spells_the_basename_of(name: &str, recipe: &ModelRecipeRecord) -> bool { + let basename = recipe + .canonical_model_id + .rsplit('/') + .next() + .unwrap_or(&recipe.canonical_model_id); + name.trim().eq_ignore_ascii_case(basename) +} + +/// Characterization test for a known false-positive surface, pinned so it is +/// visible rather than merely absent from the suite. +/// +/// Readiness accepts a model's basename for the model (an engine launched on +/// `Qwen/Qwen3-32B` commonly reports `Qwen3-32B`). Some catalog aliases spell +/// the basename of a *different* catalog model: `qwen3-32b` is an alias of +/// `Qwen/Qwen3-32B-FP8` and also the basename of `Qwen/Qwen3-32B`. An engine +/// launched on the FP8 recipe under that alias reports `qwen3-32b`, and that +/// reads as ready for `Qwen/Qwen3-32B` — readiness cannot tell the two apart +/// from the name alone. +/// +/// The collisions are derived from the catalog and compared with the list +/// below, so a catalog change that adds one fails here and has to be looked +/// at rather than being silently excluded from +/// [`listing_a_different_catalog_model_is_never_readiness`]. If readiness or +/// the catalog aliases are changed to remove a collision, update the list +/// along with them. +#[test] +fn an_alias_spelling_another_models_basename_reads_as_ready_for_it() { + let mut collisions = Vec::new(); + for owner in catalog() { + for alias in &owner.aliases { + for other in catalog() { + if other.canonical_model_id != owner.canonical_model_id + && !other.matches_ref(alias) + && spells_the_basename_of(alias, other) + { + collisions.push(( + alias.as_str(), + owner.canonical_model_id.as_str(), + other.canonical_model_id.as_str(), + )); + } + } + } + } + collisions.sort_unstable(); + assert_eq!( + collisions, + vec![ + ("qwen3-32b", "Qwen/Qwen3-32B-FP8", "Qwen/Qwen3-32B"), + ( + "qwen3.6-35b-a3b", + "unsloth/Qwen3.6-35B-A3B-GGUF:Q4_K_M", + "Qwen/Qwen3.6-35B-A3B" + ), + ], + "(alias, recipe it belongs to, recipe whose basename it spells) — the catalog's \ + alias/basename collisions changed" + ); + + for (alias, owner, other) in collisions { + assert_eq!( + endpoint_reports(models_document(&[alias]), Some(other)), + Ok(true), + "pinning the current behavior: {owner:?} served as {alias:?} reads as ready for \ + {other:?}" + ); + } +} + /// Readiness accepts a catalog model under any of its names: an engine that /// reports an alias satisfies a request for the canonical id, and the reverse. /// Checked for every alias in the shipped catalog. @@ -437,11 +493,7 @@ fn readiness_accepts_a_catalog_model_under_any_of_its_names() { (alias.as_str(), recipe.canonical_model_id.as_str()), (recipe.canonical_model_id.as_str(), alias.as_str()), ] { - let body = serde_json::json!({ - "object": "list", - "data": [{ "id": served, "object": "model" }], - }) - .to_string(); + let body = models_document(&[served]); if endpoint_reports(body, Some(wanted)) != Ok(true) { misses.push(format!("serving {served:?} did not satisfy {wanted:?}")); } @@ -489,7 +541,7 @@ fn near_miss_names(id: &str) -> Vec { /// /// Readiness matches a bare *family* word against any model with a name token /// that starts with it, and does so in both directions. One direction is -/// exercised by an existing unit test in `net.rs`: asking for `qwen` is +/// exercised by an existing unit test in `endpoint_readiness.rs`: asking for `qwen` is /// satisfied by an engine serving `Qwen3-0.6B-GGUF`. This test pins the other: an endpoint /// whose `/v1/models` reports only the bare id `qwen` — itself a catalog alias /// — is reported ready for other `Qwen*` entries in the catalog, not just the @@ -507,11 +559,7 @@ fn near_miss_names(id: &str) -> Vec { /// updated along with the behavior. #[test] fn a_bare_family_word_satisfies_readiness_for_the_whole_family() { - let body = serde_json::json!({ - "object": "list", - "data": [{ "id": "qwen", "object": "model" }], - }) - .to_string(); + let body = models_document(&["qwen"]); let cross_matched: Vec<&str> = catalog() .iter() @@ -538,7 +586,7 @@ fn a_bare_family_word_satisfies_readiness_for_the_whole_family() { "the cross-match must at least stay inside the family: {cross_matched:?}" ); - // The other direction, which the `net.rs` unit test also exercises: asking + // The other direction, which the `endpoint_readiness.rs` unit test also exercises: asking // for the bare family word is satisfied by an engine serving any catalog // model of that family. for recipe in catalog().iter().filter(|recipe| { @@ -547,11 +595,7 @@ fn a_bare_family_word_satisfies_readiness_for_the_whole_family() { .to_ascii_lowercase() .contains("qwen") }) { - let body = serde_json::json!({ - "object": "list", - "data": [{ "id": recipe.canonical_model_id, "object": "model" }], - }) - .to_string(); + let body = models_document(&[recipe.canonical_model_id.as_str()]); assert_eq!( endpoint_reports(body, Some("qwen")), Ok(true), @@ -572,11 +616,7 @@ fn a_bare_family_word_satisfies_readiness_for_an_uncatalogued_model() { ("llama", "meta-llama/Llama-3.1-70B-Instruct"), ("gemma", "google/gemma-2-27b-it"), ] { - let body = serde_json::json!({ - "object": "list", - "data": [{ "id": served, "object": "model" }], - }) - .to_string(); + let body = models_document(&[served]); assert_eq!( endpoint_reports(body, Some(wanted)), Ok(true), diff --git a/crates/rocm-engine-protocol/tests/protocol_properties.rs b/crates/rocm-engine-protocol/tests/protocol_properties.rs index 1bb43e3d3e..8aa54b9971 100644 --- a/crates/rocm-engine-protocol/tests/protocol_properties.rs +++ b/crates/rocm-engine-protocol/tests/protocol_properties.rs @@ -13,7 +13,11 @@ //! decode must produce is pinned by the round-trip, version-skew and golden //! document tests. +#[path = "../../rocm-core/tests/common/proptest_cases.rs"] +mod proptest_cases; + use proptest::prelude::*; +use proptest_cases::proptest_cases; use rocm_engine_protocol::{ DetectResponse, DevicePolicy, EngineCapabilities, EngineDeviceAvailability, EngineMethod, EngineRecipeEndpointHint, EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, @@ -30,7 +34,8 @@ macro_rules! decode_without_panic { }; } -/// Every envelope/payload type a plugin's output is decoded into. +/// Every envelope and payload type the protocol decodes, in either direction: +/// what a plugin writes (responses) and what it reads (requests). fn decode_all_without_panic(bytes: &[u8]) { decode_without_panic!(EngineRequestEnvelope, bytes); decode_without_panic!(EngineResponseEnvelope, bytes); @@ -251,7 +256,10 @@ fn mutate(json: &str, position: usize, kind: u8) -> Vec { } proptest! { - #![proptest_config(ProptestConfig { cases: 512, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(512), + ..ProptestConfig::default() + })] /// Encode then decode must be the identity for every value the type holds. #[test] @@ -365,7 +373,10 @@ proptest! { } proptest! { - #![proptest_config(ProptestConfig { cases: 1024, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(1024), + ..ProptestConfig::default() + })] /// A plugin binary name is attacker-influenced (anyone who can drop a file /// into a plugin directory). The derived engine id is later used to build @@ -392,9 +403,11 @@ proptest! { ], ) { if let Some(id) = engine_id_from_plugin_binary_name(&name) { + // The character set below has no separator, so it also rules out + // `/` and `\`. That well-formed names do yield their id, which a + // helper that never returned one would fail, is pinned by + // `plugin_binary_names_yield_their_engine_id`. prop_assert!(!id.is_empty()); - prop_assert!(!id.contains('/'), "engine id must not contain `/`: {id:?}"); - prop_assert!(!id.contains('\\'), "engine id must not contain `\\`: {id:?}"); prop_assert!(id != "." && id != "..", "engine id must not traverse: {id:?}"); prop_assert!( id.chars().all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-')), @@ -403,8 +416,11 @@ proptest! { } } - /// `--gpu` is direct user input. Parsing must never panic, and a successful - /// parse must round-trip through the CLI spelling it reports. + /// `--gpu` is direct user input. Parsing must never panic on arbitrary + /// text, and a successful parse must round-trip through the CLI spelling + /// it reports. Random text almost never parses, so this is mostly a panic + /// check; which values must parse, and to what, is pinned by + /// `gpu_cli_values_parse_as_documented`. #[test] fn gpu_selection_parse_never_panics_and_roundtrips(value in "[ -~]{0,16}") { match GpuSelection::parse_cli_value(&value) { @@ -475,7 +491,10 @@ proptest! { } proptest! { - #![proptest_config(ProptestConfig { cases: 256, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(256), + ..ProptestConfig::default() + })] /// A `LogsRequest` carries a `tail_lines` count straight from the caller into /// the engines' tail logic; it must round-trip exactly, including the @@ -545,11 +564,6 @@ proptest! { prop_assert_eq!(decoded.port, port); prop_assert_eq!(decoded.gpu_selection, selection); prop_assert_eq!(decoded.device_policy, device_policy); - prop_assert_eq!( - is_public_bind_host(&decoded.host), - is_public_bind_host(&host), - "the bind classification must not change across the wire" - ); } /// A request envelope decodes exactly when its method is one of the @@ -709,7 +723,10 @@ const HEALTHCHECK_REQUIRED_FIELDS: [&str; 5] = [ ]; proptest! { - #![proptest_config(ProptestConfig { cases: 256, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(256), + ..ProptestConfig::default() + })] /// Version skew: a newer engine sends extra fields and may omit ones this /// build requires. The document decodes exactly when every required field @@ -992,7 +1009,10 @@ fn engine_methods_use_their_documented_wire_names() { // `BTreeMap` keys in a recipe hint come off the wire; duplicates in the JSON // must resolve deterministically rather than depending on iteration order. proptest! { - #![proptest_config(ProptestConfig { cases: 256, ..ProptestConfig::default() })] + #![proptest_config(ProptestConfig { + cases: proptest_cases(256), + ..ProptestConfig::default() + })] #[test] fn duplicate_parser_setting_keys_resolve_last_wins( diff --git a/docs/testing.md b/docs/testing.md index 579f3594ca..989ac996ec 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -239,9 +239,13 @@ PROPTEST_CASES=10000 cargo test -p rocm --bin rocm proptests An explicit case count overrides `PROPTEST_CASES`, so a module that sets one reads the variable back itself: through the `proptest_cases` helper in each crate's test support, [rocm's](../apps/rocm/src/test_support.rs) and -[rocm-core's](../crates/rocm-core/src/test_support.rs), and through -`run_property` in [archive_props.rs](../crates/rocm-core/src/archive_props.rs), -which keeps its own copy because three crates include it by `#[path]`. All of +[rocm-core's](../crates/rocm-core/src/test_support.rs); through the copy in +[tests/common/proptest_cases.rs](../crates/rocm-core/tests/common/proptest_cases.rs), +which the `rocm-core` and `rocm-engine-protocol` integration tests and the +Lemonade engine's unit tests include by `#[path]` because they cannot reach a +crate's test support; and through `run_property` in +[archive_props.rs](../crates/rocm-core/src/archive_props.rs), which keeps its +own copy because three crates include it by `#[path]`. All of them only ever raise the module's count: a `PROPTEST_CASES` below it is ignored, because some of these properties catch their target defect in a small share of cases and would pass with it present on a short run. The uninstall properties read their own @@ -1852,9 +1856,13 @@ padded or re-cased, a model id with a variant suffix — because a uniformly random input is rejected by any implementation, correct or not. The pinned-key tests run the real check over a scratch repository root and walk every position of the committed key. `model_selector_properties` also pins, rather -than asserts away, that readiness accepts a bare family word such as `qwen` -reported by an engine for other models of that family, in the catalog and -beyond. +than asserts away, two shapes where readiness accepts an engine's reported name +for a different model: a bare family word such as `qwen` reads as ready for +other models of that family, in the catalog and beyond, and a catalog alias +that spells another catalog model's basename (`qwen3-32b`, an alias of +`Qwen/Qwen3-32B-FP8` and the basename of `Qwen/Qwen3-32B`) reads as ready for +that other model. The second is checked against a list of the catalog's +current collisions, so a catalog change that adds one fails the test. Model recipe engine metadata checks: diff --git a/engines/lemonade/src/archive_cache.rs b/engines/lemonade/src/archive_cache.rs index ed2805eafc..99012c2ef2 100644 --- a/engines/lemonade/src/archive_cache.rs +++ b/engines/lemonade/src/archive_cache.rs @@ -283,7 +283,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::test_support::scratch_dir; + use crate::test_support::{proptest_cases, scratch_dir}; use std::io::Write; use std::process::{Command as ProcessCommand, Stdio}; @@ -300,6 +300,25 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// Whether `error` is the digest check's own rejection, so a failure to + /// read the archive cannot pass as one. + fn is_digest_mismatch(error: &anyhow::Error) -> bool { + error.to_string().contains("SHA-256 mismatch") + } + + /// A runner with a fixed seed, so a failure reproduces as-is, running at + /// least `cases` cases. `TestRunner::deterministic` alone takes its case + /// count from `PROPTEST_CASES` unchecked, so `PROPTEST_CASES=0` would pass + /// these properties without running one. + fn deterministic_runner(cases: u32) -> proptest::test_runner::TestRunner { + let config = proptest::test_runner::Config { + cases: proptest_cases(cases), + ..proptest::test_runner::Config::default() + }; + let rng = proptest::test_runner::TestRng::deterministic_rng(config.rng_algorithm); + proptest::test_runner::TestRunner::new_with_rng(config, rng) + } + /// Property: the archive digest check accepts the pinned digest and nothing /// else. /// @@ -326,7 +345,7 @@ mod tests { assert!(verify_sha256(&archive, &truth).is_ok()); - let mut runner = proptest::test_runner::TestRunner::deterministic(); + let mut runner = deterministic_runner(256); let near_miss = { let truth = truth.clone(); (0usize..64, 1usize..64, 0u8..7).prop_map(move |(position, keep, kind)| match kind { @@ -352,7 +371,8 @@ mod tests { "the generator must only produce near misses" ); prop_assert!( - verify_sha256(&archive, &candidate).is_err(), + verify_sha256(&archive, &candidate) + .is_err_and(|error| is_digest_mismatch(&error)), "the archive digest check accepted {:?} for an archive whose digest is {:?}", candidate, truth @@ -364,7 +384,7 @@ mod tests { // An empty pin is the one near miss the generator above cannot produce // (it keeps at least one character); it must not read as "no check". assert!( - verify_sha256(&archive, "").is_err(), + verify_sha256(&archive, "").is_err_and(|error| is_digest_mismatch(&error)), "an empty pinned digest accepted the archive" ); @@ -397,18 +417,18 @@ mod tests { *tail_flipped.last_mut().unwrap() ^= 1; fs::write(&archive, &tail_flipped).unwrap(); assert!( - verify_sha256(&archive, &truth).is_err(), + verify_sha256(&archive, &truth).is_err_and(|error| is_digest_mismatch(&error)), "a flip in the last byte was accepted" ); - let mut runner = proptest::test_runner::TestRunner::deterministic(); + let mut runner = deterministic_runner(256); runner .run(&(0usize..content.len(), 0u8..8), |(position, bit)| { let mut tampered = content.clone(); tampered[position] ^= 1 << bit; fs::write(&archive, &tampered).unwrap(); prop_assert!( - verify_sha256(&archive, &truth).is_err(), + verify_sha256(&archive, &truth).is_err_and(|error| is_digest_mismatch(&error)), "an archive differing by one bit was accepted" ); Ok(()) diff --git a/engines/lemonade/src/test_support.rs b/engines/lemonade/src/test_support.rs index 7b79057c86..7a604084ef 100644 --- a/engines/lemonade/src/test_support.rs +++ b/engines/lemonade/src/test_support.rs @@ -8,6 +8,10 @@ use std::fs; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; +#[path = "../../../crates/rocm-core/tests/common/proptest_cases.rs"] +mod proptest_cases; +pub(crate) use proptest_cases::proptest_cases; + /// A fresh, empty scratch directory under the crate's `target/`, named /// `lemonade-fs-test----`. /// From 282e9466d5c78f1584c20b39913fa7070dbbaf8a Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 5 Nov 2026 16:30:16 +0200 Subject: [PATCH 04/10] test(release): make the pinned-key tests independent of key shape The drift tests asserted that the committed key's body was over 300 characters and inserted their noise before the `MII` that starts an RSA key's DER, so rotating to a key of another type or size would fail them for no defect. The coverage check now compares the walked characters with a count taken straight from the committed file, the noise goes at the start of the body, and the decoy test flips a letter on the first body line, which every key has. Signed-off-by: Roman Inflianskas --- xtask/src/verify_pinned_keys.rs | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index 41b5c9e65c..b45b6d8f0c 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -1026,10 +1026,20 @@ mod tests { } } } - let body_chars: usize = body.iter().map(|line| line.chars().count()).sum(); - assert!( - body_chars > 300, - "the walk covered only {body_chars} body characters" + // The walk covered the whole key, counted straight from the committed + // file rather than through `raw_body_lines`, whatever its size: a + // rotation to a different key type or length changes this count, not + // the test. + let walked: usize = body.iter().map(|line| line.chars().count()).sum(); + let committed: usize = canonical + .lines() + .filter(|line| !line.starts_with("-----")) + .map(|line| line.chars().count()) + .sum(); + assert!(committed > 0, "the committed key has a body"); + assert_eq!( + walked, committed, + "the walk must cover every body character of the committed key" ); assert_eq!(checked, expected_checks, "every position gets every edit"); } @@ -1142,10 +1152,10 @@ mod tests { fn a_correct_key_elsewhere_in_the_file_never_rescues_a_wrong_constant() { let canonical = canonical_release_key_text(); let mut body = raw_body_lines(&canonical); - // Flip the case of the first letter on the second body line, a + // Flip the case of the first letter on the first body line, a // one-character drift that stays inside the base64 alphabet. let flipped: String = { - let mut chars: Vec = body[1].chars().collect(); + let mut chars: Vec = body[0].chars().collect(); let at = chars .iter() .position(char::is_ascii_alphabetic) @@ -1157,7 +1167,7 @@ mod tests { }; chars.into_iter().collect() }; - body[1] = flipped; + body[0] = flipped; let wrong = pem_from_lines(&body); assert_fixture_differs(&wrong, &canonical); @@ -1219,7 +1229,13 @@ mod tests { fn non_base64_noise_in_an_embedded_key_is_normalized_away() { let canonical = canonical_release_key_text(); for noise in [" ", "!", "."] { - let noisy = canonical.replacen("\nMII", &format!("\n{noise}MII"), 1); + // At the start of the body, which every key has, rather than + // before a particular key type's leading bytes. + let noisy = canonical.replacen( + &format!("{PEM_BEGIN}\n"), + &format!("{PEM_BEGIN}\n{noise}"), + 1, + ); assert_fixture_differs(&noisy, &canonical); let root = release_layout( &canonical, From 3eab903688ff76828bd6bdecb3720b09850073aa Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 9 Nov 2026 14:25:59 +0200 Subject: [PATCH 05/10] test(support): keep one copy of the property case-count rule `rocm` and `rocm-core` each carried their own `proptest_cases`, and the integration tests added a third, so the three could drift. The shared file under `rocm-core/tests/common` now also provides `case_count_from_env`, and both crates' test support include it by `#[path]` instead of defining their own. `archive_props::run_property` keeps its inline copy, since a `#[path]` inside a file that is itself included into three crates would resolve differently in each. Signed-off-by: Roman Inflianskas --- apps/rocm/src/test_support.rs | 47 ++----------------- crates/rocm-core/src/archive_props.rs | 7 +-- crates/rocm-core/src/test_support.rs | 45 ++---------------- .../rocm-core/tests/common/proptest_cases.rs | 37 ++++++++------- docs/testing.md | 11 ++--- 5 files changed, 40 insertions(+), 107 deletions(-) diff --git a/apps/rocm/src/test_support.rs b/apps/rocm/src/test_support.rs index c724ad973f..50c351a84b 100644 --- a/apps/rocm/src/test_support.rs +++ b/apps/rocm/src/test_support.rs @@ -50,39 +50,11 @@ pub(crate) fn unique_suffix() -> String { ) } -/// Case count for a property test: `PROPTEST_CASES` (proptest's own knob, -/// which an explicit `with_cases` would otherwise override) when it is set and -/// above `default`, otherwise `default`. -/// -/// Never below `default`: the counts are chosen so a property catches the -/// defects it targets with near certainty, and some of those are hit by a -/// small share of cases, so a lower count would pass with them present. `0` -/// would pass vacuously. -pub(crate) fn proptest_cases(default: u32) -> u32 { - case_count_from_env("PROPTEST_CASES", default) -} - -/// Case count read from `var`, raising `default` but never lowering it, for -/// the same reason as [`proptest_cases`]. The uninstall properties use it with -/// their own knob, `ROCM_DELETION_PROP_CASES`. A value that is ignored, -/// because it is lower or does not parse, is reported on stderr. -pub(crate) fn case_count_from_env(var: &str, default: u32) -> u32 { - let requested = std::env::var(var).ok(); - let cases = raised_case_count(requested.as_deref(), default); - if let Some(requested) = requested - && requested.parse::().ok() != Some(cases) - { - eprintln!("{var}={requested} ignored: this property runs at least {default} cases"); - } - cases -} - -/// The pure half of [`case_count_from_env`]. -fn raised_case_count(requested: Option<&str>, default: u32) -> u32 { - requested - .and_then(|value| value.parse().ok()) - .map_or(default, |requested: u32| requested.max(default)) -} +// The case-count rule for property tests that set their own count, shared with +// `rocm-core`, its integration tests and the Lemonade engine rather than copied. +#[path = "../../../crates/rocm-core/tests/common/proptest_cases.rs"] +mod proptest_cases; +pub(crate) use proptest_cases::{case_count_from_env, proptest_cases}; /// Split `bytes` at the given (unsorted, possibly duplicated) offsets, each /// taken modulo `bytes.len() + 1`. @@ -238,13 +210,4 @@ mod tests { (0..64).map(|_| unique_suffix()).collect(); assert_eq!(suffixes.len(), 64, "two calls must never share a suffix"); } - - #[test] - fn case_counts_are_raised_but_never_lowered() { - assert_eq!(raised_case_count(None, 256), 256); - assert_eq!(raised_case_count(Some("4096"), 256), 4096); - assert_eq!(raised_case_count(Some("16"), 256), 256); - assert_eq!(raised_case_count(Some("0"), 256), 256); - assert_eq!(raised_case_count(Some("lots"), 256), 256); - } } diff --git a/crates/rocm-core/src/archive_props.rs b/crates/rocm-core/src/archive_props.rs index ec0d1f9202..8a6efd92b5 100644 --- a/crates/rocm-core/src/archive_props.rs +++ b/crates/rocm-core/src/archive_props.rs @@ -1094,9 +1094,10 @@ pub fn run_property( // `PROPTEST_CASES` (proptest's own knob, which an explicit case count // would otherwise override) scales every property up for a longer hunt. // Never down: the reach floors below are percentages, which a handful of - // cases cannot meet. The same rule as rocm-core's `proptest_cases`, kept - // inline because this file is also `#[path]`-included into `rocm` and the - // Lemonade engine, where `crate::test_support` is a different module. + // cases cannot meet. The same rule as `tests/common/proptest_cases.rs`, + // kept inline because this file is also `#[path]`-included into `rocm` and + // the Lemonade engine, and a `#[path]` inside it would resolve relative to + // each includer. let requested = std::env::var("PROPTEST_CASES").ok(); let cases = match requested.as_deref().map(str::parse::) { Some(Ok(requested)) if requested >= cases => requested, diff --git a/crates/rocm-core/src/test_support.rs b/crates/rocm-core/src/test_support.rs index 518787ee5f..4425af7965 100644 --- a/crates/rocm-core/src/test_support.rs +++ b/crates/rocm-core/src/test_support.rs @@ -178,37 +178,11 @@ pub(crate) fn audit_messages(records: &[AuditEventRecord]) -> Vec<&str> { records.iter().map(|r| r.message.as_str()).collect() } -/// Case count for a property test that sets its own count: `PROPTEST_CASES` -/// (proptest's own knob, which an explicit count would otherwise override) -/// when it is set and above `default`, otherwise `default`. -/// -/// Never below `default`: the counts are chosen so a property catches the -/// defects it targets with near certainty, and some of those are hit by a -/// small share of cases, so a lower count would pass with them present. `0` -/// would pass vacuously. A value that is ignored, because it is lower or does -/// not parse, is reported on stderr. `archive_props::run_property` applies the -/// same rule with its own copy, since other crates include that file by -/// `#[path]`. -pub(crate) fn proptest_cases(default: u32) -> u32 { - let requested = std::env::var("PROPTEST_CASES").ok(); - let cases = raised_case_count(requested.as_deref(), default); - if let Some(requested) = requested - && requested.parse::().ok() != Some(cases) - { - eprintln!( - "PROPTEST_CASES={requested} ignored: this property runs at least {default} cases" - ); - } - cases -} - -/// The pure half of [`proptest_cases`]: `requested` raises `default` and never -/// lowers it. -fn raised_case_count(requested: Option<&str>, default: u32) -> u32 { - requested - .and_then(|value| value.parse().ok()) - .map_or(default, |requested: u32| requested.max(default)) -} +// The case-count rule for property tests that set their own count, shared with +// `rocm`, the integration tests and the Lemonade engine rather than copied. +#[path = "../tests/common/proptest_cases.rs"] +mod proptest_cases; +pub(crate) use proptest_cases::proptest_cases; #[cfg(test)] mod tests { @@ -218,13 +192,4 @@ mod tests { fn temp_app_paths_gives_each_call_its_own_root_even_for_one_label() { assert_each_call_gets_its_own_root(|| temp_app_paths("same-label").0); } - - #[test] - fn proptest_cases_are_raised_but_never_lowered() { - assert_eq!(raised_case_count(None, 256), 256); - assert_eq!(raised_case_count(Some("4096"), 256), 4096); - assert_eq!(raised_case_count(Some("16"), 256), 256); - assert_eq!(raised_case_count(Some("0"), 256), 256); - assert_eq!(raised_case_count(Some("lots"), 256), 256); - } } diff --git a/crates/rocm-core/tests/common/proptest_cases.rs b/crates/rocm-core/tests/common/proptest_cases.rs index 177fdfa460..50b3090e5d 100644 --- a/crates/rocm-core/tests/common/proptest_cases.rs +++ b/crates/rocm-core/tests/common/proptest_cases.rs @@ -2,37 +2,42 @@ // // SPDX-License-Identifier: MIT -//! The case-count rule for property tests that set their own count, shared by -//! the `rocm-core` and `rocm-engine-protocol` integration tests and the -//! Lemonade engine's unit tests, which include this file by `#[path]`. -//! -//! It is the same rule as `rocm-core`'s crate-internal -//! `test_support::proptest_cases`, which integration tests and other crates -//! cannot reach. +//! The case-count rule for property tests that set their own count. This file +//! is its one implementation: `rocm`'s and `rocm-core`'s crate-internal +//! `test_support`, the `rocm-core` and `rocm-engine-protocol` integration +//! tests, and the Lemonade engine's unit tests all include it by `#[path]`. +//! `archive_props::run_property` applies the same rule inline, since that file +//! is itself included by `#[path]` into three crates. /// Case count for a property test: `PROPTEST_CASES` (proptest's own knob, /// which an explicit count would otherwise override) when it is set and above /// `default`, otherwise `default`. /// /// Never below `default`: each count is chosen so its property reaches the -/// cases it targets, and `PROPTEST_CASES=0` would otherwise pass every -/// property without running one. A value that is ignored, because it is lower -/// or does not parse, is reported on stderr. +/// cases it targets, and some of those are hit by a small share of cases, so a +/// lower count would pass with the defect present. `PROPTEST_CASES=0` would +/// otherwise pass every property without running one. pub fn proptest_cases(default: u32) -> u32 { - let requested = std::env::var("PROPTEST_CASES").ok(); + case_count_from_env("PROPTEST_CASES", default) +} + +/// Case count read from `var`, raising `default` but never lowering it, for +/// the same reason as [`proptest_cases`]. `rocm`'s uninstall properties use it +/// with their own knob, `ROCM_DELETION_PROP_CASES`. A value that is ignored, +/// because it is lower or does not parse, is reported on stderr. +pub fn case_count_from_env(var: &str, default: u32) -> u32 { + let requested = std::env::var(var).ok(); let cases = raised_case_count(requested.as_deref(), default); if let Some(requested) = requested && requested.parse::().ok() != Some(cases) { - eprintln!( - "PROPTEST_CASES={requested} ignored: this property runs at least {default} cases" - ); + eprintln!("{var}={requested} ignored: this property runs at least {default} cases"); } cases } -/// The pure half of [`proptest_cases`]: `requested` raises `default` and never -/// lowers it. +/// The pure half of [`case_count_from_env`]: `requested` raises `default` and +/// never lowers it. pub fn raised_case_count(requested: Option<&str>, default: u32) -> u32 { requested .and_then(|value| value.parse().ok()) diff --git a/docs/testing.md b/docs/testing.md index 989ac996ec..8b0ba36af8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -237,13 +237,12 @@ PROPTEST_CASES=10000 cargo test -p rocm --bin rocm proptests ``` An explicit case count overrides `PROPTEST_CASES`, so a module that sets one -reads the variable back itself: through the `proptest_cases` helper in each -crate's test support, [rocm's](../apps/rocm/src/test_support.rs) and -[rocm-core's](../crates/rocm-core/src/test_support.rs); through the copy in +reads the variable back itself: through the `proptest_cases` helper in [tests/common/proptest_cases.rs](../crates/rocm-core/tests/common/proptest_cases.rs), -which the `rocm-core` and `rocm-engine-protocol` integration tests and the -Lemonade engine's unit tests include by `#[path]` because they cannot reach a -crate's test support; and through `run_property` in +which [rocm's](../apps/rocm/src/test_support.rs) and +[rocm-core's](../crates/rocm-core/src/test_support.rs) test support, the +`rocm-core` and `rocm-engine-protocol` integration tests and the Lemonade +engine's unit tests all include by `#[path]`; and through `run_property` in [archive_props.rs](../crates/rocm-core/src/archive_props.rs), which keeps its own copy because three crates include it by `#[path]`. All of them only ever raise the module's count: a `PROPTEST_CASES` below it is From 54714061518082a7db941224ca8569df13f6ab8a Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 9 Nov 2026 14:27:08 +0200 Subject: [PATCH 06/10] test(integrity): size the multi-chunk body from the real chunk size The digest gate test hand-copied the private `DOWNLOAD_CHUNK_BYTES`, so the two could drift with nothing to notice. The constant is now public beside `DOWNLOAD_MAX_ATTEMPTS`, and the test builds its three-chunks- plus-a-byte body from it. Signed-off-by: Roman Inflianskas --- crates/rocm-core/src/download.rs | 10 +++++++--- crates/rocm-core/src/lib.rs | 4 ++-- crates/rocm-core/tests/digest_gate_properties.rs | 15 +++++++-------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/crates/rocm-core/src/download.rs b/crates/rocm-core/src/download.rs index 81f2c7d84d..f27aa793cd 100644 --- a/crates/rocm-core/src/download.rs +++ b/crates/rocm-core/src/download.rs @@ -26,9 +26,13 @@ use crate::disk_space; /// retry, so this is one initial try plus two retries. pub const DOWNLOAD_MAX_ATTEMPTS: u32 = 3; -/// Chunk size for the streaming copy. Fixed, so peak memory is independent of -/// the artifact size — the whole point of streaming a multi-gigabyte tarball. -const DOWNLOAD_CHUNK_BYTES: usize = 64 * 1024; +/// Chunk size for the streaming copy. +/// +/// Fixed, so peak memory is independent of the artifact size — the whole point +/// of streaming a multi-gigabyte tarball. Public so the digest gate's tests can +/// size a body that crosses a chunk boundary from this value rather than from a +/// copy of it. +pub const DOWNLOAD_CHUNK_BYTES: usize = 64 * 1024; /// Exponential backoff between download attempts. /// diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 69e2da9599..024ebccdbd 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -99,8 +99,8 @@ pub use disk_space::{ mount_for_path, on_same_mount, warn_if_low_space, with_margin, }; pub use download::{ - Backoff, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, DownloadRequest, download_file_streaming, - download_file_streaming_with_progress, download_file_to_path, + Backoff, DOWNLOAD_CHUNK_BYTES, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, DownloadRequest, + download_file_streaming, download_file_streaming_with_progress, download_file_to_path, download_file_to_path_with_progress, }; pub use endpoint_key::{endpoint_api_key_has_forbidden_chars, generate_endpoint_api_key}; diff --git a/crates/rocm-core/tests/digest_gate_properties.rs b/crates/rocm-core/tests/digest_gate_properties.rs index 93a0a2f22b..beea93c0c9 100644 --- a/crates/rocm-core/tests/digest_gate_properties.rs +++ b/crates/rocm-core/tests/digest_gate_properties.rs @@ -24,7 +24,7 @@ use std::time::Duration; use loopback::LoopbackServer; use proptest::prelude::*; use proptest_cases::proptest_cases; -use rocm_core::{DownloadRequest, download_file_streaming}; +use rocm_core::{DOWNLOAD_CHUNK_BYTES, DownloadRequest, download_file_streaming}; use sha2::{Digest, Sha256}; fn scratch_dir(tag: &str) -> PathBuf { @@ -94,12 +94,6 @@ fn is_digest_rejection(result: &Result<(), String>) -> bool { .is_err_and(|error| error.contains("SHA-256 mismatch")) } -/// The chunk size the download streams through, copied from the private -/// `DOWNLOAD_CHUNK_BYTES` in `download.rs`. The multi-chunk body below spans three -/// of these plus a byte, so it still crosses a chunk boundary if that constant -/// grows a little, and regardless of how short the socket reads come back. -const CHUNK_BYTES: usize = 64 * 1024; - fn digest_of(body: &[u8]) -> String { format!("{:x}", Sha256::digest(body)) } @@ -297,7 +291,12 @@ proptest! { /// is skipped or disabled for a body this large.) #[test] fn tampering_past_the_first_chunk_is_rejected() { - let body: Vec = (0..=255_u8).cycle().take(3 * CHUNK_BYTES + 1).collect(); + // Three chunks plus a byte, so the body crosses a chunk boundary + // regardless of how short the socket reads come back. + let body: Vec = (0..=255_u8) + .cycle() + .take(3 * DOWNLOAD_CHUNK_BYTES + 1) + .collect(); let truth = digest_of(&body); assert!( download_accepts(&body, Some(&truth), "multi-chunk-intact"), From b66cf1f868e48d255377bb8491c746fded91e4d6 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 9 Nov 2026 14:33:01 +0200 Subject: [PATCH 07/10] test(integrity): remove the digest gate's scratch dir on failure too The scratch directory was removed only after the assertions passed, so a failing assertion left it in the system temp dir. It is now a `tempfile::TempDir`, which is removed when it drops, including while a panic unwinds. Signed-off-by: Roman Inflianskas --- .../rocm-core/tests/digest_gate_properties.rs | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/crates/rocm-core/tests/digest_gate_properties.rs b/crates/rocm-core/tests/digest_gate_properties.rs index beea93c0c9..c0aef3ca0f 100644 --- a/crates/rocm-core/tests/digest_gate_properties.rs +++ b/crates/rocm-core/tests/digest_gate_properties.rs @@ -17,8 +17,6 @@ mod loopback; #[path = "common/proptest_cases.rs"] mod proptest_cases; -use std::path::PathBuf; -use std::thread; use std::time::Duration; use loopback::LoopbackServer; @@ -27,15 +25,13 @@ use proptest_cases::proptest_cases; use rocm_core::{DOWNLOAD_CHUNK_BYTES, DownloadRequest, download_file_streaming}; use sha2::{Digest, Sha256}; -fn scratch_dir(tag: &str) -> PathBuf { - let dir = std::env::temp_dir().join(format!( - "rocm-core-digest-gate-{tag}-{}-{:?}", - std::process::id(), - thread::current().id() - )); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).expect("scratch dir"); - dir +/// A fresh scratch directory, removed when the returned guard drops — including +/// when an assertion fails, so a failing run leaves nothing in the temp dir. +fn scratch_dir(tag: &str) -> tempfile::TempDir { + tempfile::Builder::new() + .prefix(&format!("rocm-core-digest-gate-{tag}-")) + .tempdir() + .expect("scratch dir") } /// Run one download against a loopback server, returning whether the gate @@ -55,7 +51,8 @@ fn download_outcome( expected_sha256: Option<&str>, tag: &str, ) -> (Result<(), String>, Vec) { - let dir = scratch_dir(tag); + let scratch = scratch_dir(tag); + let dir = scratch.path(); let destination = dir.join("artifact.bin"); let server = LoopbackServer::start(None, body.to_vec()); let url = format!("{}/artifact.bin", server.url()); @@ -71,7 +68,7 @@ fn download_outcome( "the download never reached the test server: {result:?}" ); drop(server); - let mut left_behind: Vec = std::fs::read_dir(&dir) + let mut left_behind: Vec = std::fs::read_dir(dir) .expect("read scratch dir") .map(|entry| { entry @@ -82,7 +79,6 @@ fn download_outcome( }) .collect(); left_behind.sort(); - let _ = std::fs::remove_dir_all(&dir); (result, left_behind) } From ded6d178bb314bf9e411147fdda49e05867383fd Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 9 Nov 2026 14:36:30 +0200 Subject: [PATCH 08/10] test(release): say what the pinned-key check's punctuation gap costs The check compares only each key's base64 alphabet, and a test pins that a space, `!` or `.` inserted into an installer's copy still passes. A space is harmless to both installers, but OpenSSL refuses to load the key with `!` or `.` in it, so such an installer rejects every release signed with that key while the check passes. The test's comment and docs/testing.md now say so, and what a tightening would change, so that change knows to update the test. The check itself is unchanged. Signed-off-by: Roman Inflianskas --- docs/testing.md | 6 ++++++ xtask/src/verify_pinned_keys.rs | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 8b0ba36af8..a31b8a6e64 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1862,6 +1862,12 @@ that spells another catalog model's basename (`qwen3-32b`, an alias of `Qwen/Qwen3-32B-FP8` and the basename of `Qwen/Qwen3-32B`) reads as ready for that other model. The second is checked against a list of the catalog's current collisions, so a catalog change that adds one fails the test. +The pinned-key tests likewise pin one looseness of the check itself: it +compares only the base64 alphabet of each embedded key, so punctuation such as +`!` or `.` inserted into an installer's copy is not reported, although OpenSSL +then refuses to load that key and the installer rejects every release signed +with it. `non_base64_noise_in_an_embedded_key_is_normalized_away` holds that +behaviour, so a change that tightens the comparison has to update it. Model recipe engine metadata checks: diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index b45b6d8f0c..b68e47a28b 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -1221,10 +1221,20 @@ mod tests { /// embedded key's body — whitespace or punctuation — are normalized away /// and do not register as divergence. /// - /// Whether such a body is harmless rests on the consumers' PEM readers - /// (OpenSSL in the installers, `pem-rfc7468` in the binary) rejecting or - /// ignoring it at load time; this check does not decide that. Pinned so the - /// normalization's reach is visible rather than implicit. + /// The two kinds differ for the installers that read the key. Whitespace + /// is harmless: OpenSSL, which install.sh calls, loads the key with a space + /// inserted, and `Convert.FromBase64String` (install.ps1) skips whitespace. + /// Punctuation is not: OpenSSL refuses to load the key with a `!` or `.` + /// inserted, and `FromBase64String` throws on any other character outside + /// the alphabet. Such an installer rejects every release signed with that + /// key — it fails closed, not open — while this check passes, which is the + /// drift between CI and the installers that the check exists to catch. + /// + /// Not changed here. Tightening means keeping every non-whitespace + /// character in the comparison while still dropping what each embedding's + /// quoting adds (the `\n` and `` `n `` escapes, a Rust string's `\` line + /// continuations); that change should turn the `!` and `.` cases below into + /// divergence and update this test. #[test] fn non_base64_noise_in_an_embedded_key_is_normalized_away() { let canonical = canonical_release_key_text(); From 36b12e2e49bf32a8e881e771d919cfbd5bed7fd8 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 9 Nov 2026 16:34:23 +0200 Subject: [PATCH 09/10] test(support): re-export the env-knob case count only on Unix Its one caller, the uninstall properties, is Unix-only, so on Windows the re-export was an unused import and failed the build under `-D warnings`. Signed-off-by: Roman Inflianskas --- apps/rocm/src/test_support.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/rocm/src/test_support.rs b/apps/rocm/src/test_support.rs index 50c351a84b..c25ccec899 100644 --- a/apps/rocm/src/test_support.rs +++ b/apps/rocm/src/test_support.rs @@ -54,7 +54,10 @@ pub(crate) fn unique_suffix() -> String { // `rocm-core`, its integration tests and the Lemonade engine rather than copied. #[path = "../../../crates/rocm-core/tests/common/proptest_cases.rs"] mod proptest_cases; -pub(crate) use proptest_cases::{case_count_from_env, proptest_cases}; +pub(crate) use proptest_cases::proptest_cases; +// Only the Unix-only uninstall properties read their own knob. +#[cfg(unix)] +pub(crate) use proptest_cases::case_count_from_env; /// Split `bytes` at the given (unsorted, possibly duplicated) offsets, each /// taken modulo `bytes.len() + 1`. From 4f97275b5ea746be8b88fa57ce9a62b5e60d8377 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 9 Nov 2026 17:32:26 +0200 Subject: [PATCH 10/10] test(release): match the pinned-key tests to the checks main now enforces The pinned-key tests built installers that main's check (#830) no longer accepts, so they failed once it landed: - the fake install.sh has no `readonly` line for its slots; - a dormant slot (blank canonical key) must be empty in every installer, and an empty PowerShell slot is the one-line `""` form; - the layout has no Rust source for the metadata key, which the check reads even while that entry is dormant; - a missing constant is reported as a missing assignment, not as divergence. The punctuation test documented a gap the check no longer has: a space, `!` or `.` inserted into an installer's copy of the key is now refused as a malformed key block in either installer. The test and docs/testing.md now pin the refusal. Signed-off-by: Roman Inflianskas --- docs/testing.md | 11 ++- xtask/src/verify_pinned_keys.rs | 143 ++++++++++++++++++++------------ 2 files changed, 93 insertions(+), 61 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index c3be8e3f2b..c93eca4b76 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2433,12 +2433,11 @@ that spells another catalog model's basename (`qwen3-32b`, an alias of `Qwen/Qwen3-32B-FP8` and the basename of `Qwen/Qwen3-32B`) reads as ready for that other model. The second is checked against a list of the catalog's current collisions, so a catalog change that adds one fails the test. -The pinned-key tests likewise pin one looseness of the check itself: it -compares only the base64 alphabet of each embedded key, so punctuation such as -`!` or `.` inserted into an installer's copy is not reported, although OpenSSL -then refuses to load that key and the installer rejects every release signed -with it. `non_base64_noise_in_an_embedded_key_is_normalized_away` holds that -behaviour, so a change that tightens the comparison has to update it. +The pinned-key tests also pin that a character outside the base64 alphabet +(a space, `!` or `.`) inserted into an installer's copy of the key is refused +as a malformed key block, in either installer, rather than ignored by the +comparison: an installer that embeds such a key cannot load it. +`non_base64_noise_in_an_embedded_key_is_refused_as_malformed` holds that. Model recipe engine metadata checks: diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index 24fa4c9e9d..752cf97a76 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -2450,30 +2450,42 @@ mod tests { } /// The two installer constants the current release key is pinned in, in the - /// shape the real installers embed them. + /// shape the real installers embed them, including the `readonly` line the + /// check requires after the assignments. fn install_sh(current: &str, extra: &str) -> String { format!( "#!/bin/sh\n{extra}PINNED_RELEASE_PUBLIC_KEY_CURRENT=\"{current}\"\n\ - PINNED_RELEASE_PUBLIC_KEY_NEXT=\"\"\n" + PINNED_RELEASE_PUBLIC_KEY_NEXT=\"\"\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\n" ) } fn install_ps1(current: &str, extra: &str) -> String { - format!( - "{extra}$PinnedReleasePublicKeyCurrent = @\"\n{current}\"@\n\ - $PinnedReleasePublicKeyNext = \"\"\n" - ) + // An empty slot is the one-line `""` form, as in a dormant installer; an + // empty here-string is a malformed key block, not an empty slot. + let slot = if current.is_empty() { + "$PinnedReleasePublicKeyCurrent = \"\"\n".to_owned() + } else { + format!("$PinnedReleasePublicKeyCurrent = @\"\n{current}\"@\n") + }; + format!("{extra}{slot}$PinnedReleasePublicKeyNext = \"\"\n") } /// A repository root holding the real canonical current release key and the /// given installer sources, for running [`check_pinned_keys`] — the real /// comparison — end to end. The other canonical keys are absent, so their - /// entries are dormant. + /// entries are dormant; the check still reads the metadata key's Rust + /// source to confirm its slot is empty, so the layout holds one. fn release_layout(canonical: &str, sh: &str, ps1: &str) -> tempfile::TempDir { let dir = tempfile::tempdir().expect("tempdir"); std::fs::create_dir_all(dir.path().join("docs/keys")).expect("create docs/keys"); std::fs::write(dir.path().join(PINNED_KEYS[0].canonical), canonical) .expect("write canonical key"); + let rust = dir.path().join(rs_path()); + std::fs::create_dir_all(rust.parent().expect("a parent")) + .expect("create the Rust source dir"); + std::fs::write(rust, "const PINNED_METADATA_PUBLIC_KEY_PEM: &str = \"\";\n") + .expect("write the dormant Rust source"); std::fs::write(dir.path().join("install.sh"), sh).expect("write install.sh"); std::fs::write(dir.path().join("install.ps1"), ps1).expect("write install.ps1"); dir @@ -2644,11 +2656,12 @@ mod tests { } /// A published canonical key whose pinned constant is empty, or missing - /// from the source altogether, is divergence — not a match and not a - /// dormant skip. Each case leaves exactly one entry wrong, so the error - /// must name that entry. + /// from the source altogether, is an error — not a match and not a dormant + /// skip. An empty constant is divergence; a missing one is reported as a + /// missing assignment. Each case leaves exactly one entry wrong, so the + /// error must name that entry. #[test] - fn an_empty_or_missing_constant_against_a_published_key_diverges() { + fn an_empty_or_missing_constant_against_a_published_key_is_an_error() { let canonical = canonical_release_key_text(); let root = release_layout( &canonical, @@ -2664,6 +2677,7 @@ mod tests { install_ps1(&canonical, ""), false, "release-current", + "have diverged", ), ( "current empty in install.ps1", @@ -2671,6 +2685,7 @@ mod tests { install_ps1("", ""), false, "release-current", + "have diverged", ), ( "current constant missing from install.sh", @@ -2678,6 +2693,7 @@ mod tests { install_ps1(&canonical, ""), false, "release-current", + "has no assignment of PINNED_RELEASE_PUBLIC_KEY_CURRENT", ), // The rotation case: the next key's PEM is published (the same key // stands in for it) while the installers still carry `NEXT=""`. @@ -2687,9 +2703,10 @@ mod tests { install_ps1(&canonical, ""), true, "release-next", + "have diverged", ), ]; - for (label, sh, ps1, publish_next, entry) in cases { + for (label, sh, ps1, publish_next, entry, reported_as) in cases { if publish_next { std::fs::write(&next_canonical, &canonical).expect("write canonical next key"); } else { @@ -2697,14 +2714,17 @@ mod tests { } write_sources(root.path(), &sh, &ps1); let result = check_pinned_keys(root.path(), None); + let message = result + .as_ref() + .expect_err(&format!("{label}: expected an error, got {result:?}")) + .to_string(); assert!( - diverges(&result), - "{label}: expected divergence, got {result:?}" + message.contains(reported_as), + "{label}: expected the error to say `{reported_as}`, got {message}" ); - let message = result.expect_err("diverged").to_string(); assert!( message.starts_with(&format!("{entry}:")), - "{label}: the divergence must be reported for {entry}, got {message}" + "{label}: the error must be reported for {entry}, got {message}" ); } } @@ -2715,22 +2735,24 @@ mod tests { #[test] fn a_blank_canonical_key_is_dormant_and_a_missing_source_is_an_error() { let canonical = canonical_release_key_text(); - let root = release_layout( - " \n", - &install_sh(&canonical, ""), - &install_ps1(&canonical, ""), - ); + // A dormant slot is empty in every installer; a key pasted into one is + // its own error (`a_dormant_slot_must_be_empty`). + let root = release_layout(" \n", &install_sh("", ""), &install_ps1("", "")); let messages = check_pinned_keys(root.path(), None).expect("a blank canonical key is dormant"); assert!( - messages - .iter() - .any(|m| m == "release-current: no canonical key yet — skipped (dormant)"), + messages.iter().any(|m| m + == "release-current: no canonical key yet — slots empty in install.sh, \ + install.ps1 (dormant)"), "a blank canonical key must be reported dormant, got {messages:?}" ); + // Publish the key and pin it in install.sh, so the missing install.ps1 + // is the only fault left. std::fs::write(root.path().join(PINNED_KEYS[0].canonical), &canonical) .expect("write canonical key"); + std::fs::write(root.path().join("install.sh"), install_sh(&canonical, "")) + .expect("pin the key in install.sh"); std::fs::remove_file(root.path().join("install.ps1")).expect("remove install.ps1"); let error = check_pinned_keys(root.path(), None) .expect_err("a missing pinned source must be an error"); @@ -2815,45 +2837,56 @@ mod tests { assert_ne!(left, right, "the fixture must actually differ"); } - /// Characterization: the comparison reduces each key to its base64 - /// alphabet, so characters *outside* that alphabet inserted into an - /// embedded key's body — whitespace or punctuation — are normalized away - /// and do not register as divergence. - /// - /// The two kinds differ for the installers that read the key. Whitespace - /// is harmless: OpenSSL, which install.sh calls, loads the key with a space - /// inserted, and `Convert.FromBase64String` (install.ps1) skips whitespace. - /// Punctuation is not: OpenSSL refuses to load the key with a `!` or `.` - /// inserted, and `FromBase64String` throws on any other character outside - /// the alphabet. Such an installer rejects every release signed with that - /// key — it fails closed, not open — while this check passes, which is the - /// drift between CI and the installers that the check exists to catch. + /// A character outside the base64 alphabet inserted into an installer's + /// copy of the key — whitespace or punctuation — is refused as a malformed + /// key block, in either installer, and the error names the entry and the + /// file. /// - /// Not changed here. Tightening means keeping every non-whitespace - /// character in the comparison while still dropping what each embedding's - /// quoting adds (the `\n` and `` `n `` escapes, a Rust string's `\` line - /// continuations); that change should turn the `!` and `.` cases below into - /// divergence and update this test. + /// The comparison itself only looks at the base64 alphabet, so this is the + /// well-formedness rule (`pinned_pem_body`) doing the work. It has to: an + /// installer that embeds such a key cannot load it (OpenSSL refuses a `!` or + /// `.`, and `Convert.FromBase64String` throws on anything outside the + /// alphabet), so it would reject every release signed with that key while a + /// comparison alone would pass. #[test] - fn non_base64_noise_in_an_embedded_key_is_normalized_away() { + fn non_base64_noise_in_an_embedded_key_is_refused_as_malformed() { let canonical = canonical_release_key_text(); for noise in [" ", "!", "."] { - // At the start of the body, which every key has, rather than - // before a particular key type's leading bytes. + // After the first character of the body, which every key has, rather + // than at the start of a line: install.sh refuses a key line that + // starts with whitespace, and that is a separate rule. + let first = canonical + .strip_prefix(&format!("{PEM_BEGIN}\n")) + .and_then(|body| body.chars().next()) + .expect("a body character after the BEGIN line"); let noisy = canonical.replacen( - &format!("{PEM_BEGIN}\n"), - &format!("{PEM_BEGIN}\n{noise}"), + &format!("{PEM_BEGIN}\n{first}"), + &format!("{PEM_BEGIN}\n{first}{noise}"), 1, ); assert_fixture_differs(&noisy, &canonical); - let root = release_layout( - &canonical, - &install_sh(&noisy, ""), - &install_ps1(&noisy, ""), - ); - check_pinned_keys(root.path(), None).unwrap_or_else(|error| { - panic!("{noise:?} inside the body is expected to be normalized away: {error:#}") - }); + for (file, sh, ps1) in [ + ( + "install.sh", + install_sh(&noisy, ""), + install_ps1(&canonical, ""), + ), + ( + "install.ps1", + install_sh(&canonical, ""), + install_ps1(&noisy, ""), + ), + ] { + let root = release_layout(&canonical, &sh, &ps1); + let error = check_pinned_keys(root.path(), None) + .expect_err("a key with stray characters in its body must be refused") + .to_string(); + assert!( + error.starts_with(&format!("release-current: {file} ")) + && error.contains("is not one well-formed PUBLIC KEY PEM block"), + "{noise:?} in {file}: expected a malformed-block error naming {file}, got {error}" + ); + } } } }