diff --git a/Cargo.lock b/Cargo.lock index 09d0937413..9dcc82a891 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3904,6 +3904,7 @@ dependencies = [ name = "rocm-engine-protocol" version = "0.1.0" dependencies = [ + "proptest", "rocm-core", "serde", "serde_json", diff --git a/apps/rocm/src/test_support.rs b/apps/rocm/src/test_support.rs index 1987de3679..f1ceb498d4 100644 --- a/apps/rocm/src/test_support.rs +++ b/apps/rocm/src/test_support.rs @@ -50,39 +50,14 @@ pub(crate) fn unique_suffix() -> String { ) } -/// Case count for a property test: `PROPTEST_CASES` (proptest's own knob, -/// which an explicit `with_cases` would otherwise override) when it is set and -/// above `default`, otherwise `default`. -/// -/// Never below `default`: the counts are chosen so a property catches the -/// defects it targets with near certainty, and some of those are hit by a -/// small share of cases, so a lower count would pass with them present. `0` -/// would pass vacuously. -pub(crate) fn proptest_cases(default: u32) -> u32 { - case_count_from_env("PROPTEST_CASES", default) -} - -/// Case count read from `var`, raising `default` but never lowering it, for -/// the same reason as [`proptest_cases`]. The uninstall properties use it with -/// their own knob, `ROCM_DELETION_PROP_CASES`. A value that is ignored, -/// because it is lower or does not parse, is reported on stderr. -pub(crate) fn case_count_from_env(var: &str, default: u32) -> u32 { - let requested = std::env::var(var).ok(); - let cases = raised_case_count(requested.as_deref(), default); - if let Some(requested) = requested - && requested.parse::().ok() != Some(cases) - { - eprintln!("{var}={requested} ignored: this property runs at least {default} cases"); - } - cases -} - -/// The pure half of [`case_count_from_env`]. -fn raised_case_count(requested: Option<&str>, default: u32) -> u32 { - requested - .and_then(|value| value.parse().ok()) - .map_or(default, |requested: u32| requested.max(default)) -} +// The case-count rule for property tests that set their own count, shared with +// `rocm-core`, its integration tests and the Lemonade engine rather than copied. +#[path = "../../../crates/rocm-core/tests/common/proptest_cases.rs"] +mod proptest_cases; +pub(crate) use proptest_cases::proptest_cases; +// Only the Unix-only uninstall properties read their own knob. +#[cfg(unix)] +pub(crate) use proptest_cases::case_count_from_env; /// Split `bytes` at the given (unsorted, possibly duplicated) offsets, each /// taken modulo `bytes.len() + 1`. @@ -475,13 +450,4 @@ mod tests { (0..64).map(|_| unique_suffix()).collect(); assert_eq!(suffixes.len(), 64, "two calls must never share a suffix"); } - - #[test] - fn case_counts_are_raised_but_never_lowered() { - assert_eq!(raised_case_count(None, 256), 256); - assert_eq!(raised_case_count(Some("4096"), 256), 4096); - assert_eq!(raised_case_count(Some("16"), 256), 256); - assert_eq!(raised_case_count(Some("0"), 256), 256); - assert_eq!(raised_case_count(Some("lots"), 256), 256); - } } diff --git a/crates/rocm-core/Cargo.toml b/crates/rocm-core/Cargo.toml index 48afe6fee4..cdb9855593 100644 --- a/crates/rocm-core/Cargo.toml +++ b/crates/rocm-core/Cargo.toml @@ -45,7 +45,10 @@ ureq = { version = "2.12", features = ["native-certs"] } # # Also used by `disk_space`, for the same reason: its unit formatting, margin # arithmetic and mount selection are small total functions whose contracts are -# easy to state over ALL inputs and awkward to cover with examples. +# easy to state over ALL inputs and awkward to cover with examples. And by the +# integration suites under `tests/` (download digest gate, signature +# verification, model selection and readiness), whose generators aim at near +# misses no example list enumerates. # Test-only, so it adds nothing to any shipped binary. proptest = { version = "1", default-features = false, features = ["std"] } # Archive-extraction property tests (`src/archive_props.rs`) build hostile diff --git a/crates/rocm-core/src/archive_props.rs b/crates/rocm-core/src/archive_props.rs index ec0d1f9202..8a6efd92b5 100644 --- a/crates/rocm-core/src/archive_props.rs +++ b/crates/rocm-core/src/archive_props.rs @@ -1094,9 +1094,10 @@ pub fn run_property( // `PROPTEST_CASES` (proptest's own knob, which an explicit case count // would otherwise override) scales every property up for a longer hunt. // Never down: the reach floors below are percentages, which a handful of - // cases cannot meet. The same rule as rocm-core's `proptest_cases`, kept - // inline because this file is also `#[path]`-included into `rocm` and the - // Lemonade engine, where `crate::test_support` is a different module. + // cases cannot meet. The same rule as `tests/common/proptest_cases.rs`, + // kept inline because this file is also `#[path]`-included into `rocm` and + // the Lemonade engine, and a `#[path]` inside it would resolve relative to + // each includer. let requested = std::env::var("PROPTEST_CASES").ok(); let cases = match requested.as_deref().map(str::parse::) { Some(Ok(requested)) if requested >= cases => requested, diff --git a/crates/rocm-core/src/download.rs b/crates/rocm-core/src/download.rs index 9dd6cc8c80..bd9aad7ed5 100644 --- a/crates/rocm-core/src/download.rs +++ b/crates/rocm-core/src/download.rs @@ -26,9 +26,13 @@ use crate::disk_space; /// retry, so this is one initial try plus two retries. pub const DOWNLOAD_MAX_ATTEMPTS: u32 = 3; -/// Chunk size for the streaming copy. Fixed, so peak memory is independent of -/// the artifact size — the whole point of streaming a multi-gigabyte tarball. -const DOWNLOAD_CHUNK_BYTES: usize = 64 * 1024; +/// Chunk size for the streaming copy. +/// +/// Fixed, so peak memory is independent of the artifact size — the whole point +/// of streaming a multi-gigabyte tarball. Public so the digest gate's tests can +/// size a body that crosses a chunk boundary from this value rather than from a +/// copy of it. +pub const DOWNLOAD_CHUNK_BYTES: usize = 64 * 1024; /// Exponential backoff between download attempts. /// diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 0ccf9962e9..cb8f1180b7 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -108,8 +108,8 @@ pub use disk_space::{ mount_for_path, on_same_mount, warn_if_low_space, with_margin, }; pub use download::{ - Backoff, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, DownloadRequest, download_file_streaming, - download_file_streaming_with_progress, download_file_to_path, + Backoff, DOWNLOAD_CHUNK_BYTES, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, DownloadRequest, + download_file_streaming, download_file_streaming_with_progress, download_file_to_path, download_file_to_path_with_progress, }; pub use endpoint_key::{endpoint_api_key_has_forbidden_chars, generate_endpoint_api_key}; diff --git a/crates/rocm-core/src/test_support.rs b/crates/rocm-core/src/test_support.rs index f5f2c18410..b8f94280a6 100644 --- a/crates/rocm-core/src/test_support.rs +++ b/crates/rocm-core/src/test_support.rs @@ -179,37 +179,11 @@ pub(crate) fn audit_messages(records: &[AuditEventRecord]) -> Vec<&str> { records.iter().map(|r| r.message.as_str()).collect() } -/// Case count for a property test that sets its own count: `PROPTEST_CASES` -/// (proptest's own knob, which an explicit count would otherwise override) -/// when it is set and above `default`, otherwise `default`. -/// -/// Never below `default`: the counts are chosen so a property catches the -/// defects it targets with near certainty, and some of those are hit by a -/// small share of cases, so a lower count would pass with them present. `0` -/// would pass vacuously. A value that is ignored, because it is lower or does -/// not parse, is reported on stderr. `archive_props::run_property` applies the -/// same rule with its own copy, since other crates include that file by -/// `#[path]`. -pub(crate) fn proptest_cases(default: u32) -> u32 { - let requested = std::env::var("PROPTEST_CASES").ok(); - let cases = raised_case_count(requested.as_deref(), default); - if let Some(requested) = requested - && requested.parse::().ok() != Some(cases) - { - eprintln!( - "PROPTEST_CASES={requested} ignored: this property runs at least {default} cases" - ); - } - cases -} - -/// The pure half of [`proptest_cases`]: `requested` raises `default` and never -/// lowers it. -fn raised_case_count(requested: Option<&str>, default: u32) -> u32 { - requested - .and_then(|value| value.parse().ok()) - .map_or(default, |requested: u32| requested.max(default)) -} +// The case-count rule for property tests that set their own count, shared with +// `rocm`, the integration tests and the Lemonade engine rather than copied. +#[path = "../tests/common/proptest_cases.rs"] +mod proptest_cases; +pub(crate) use proptest_cases::proptest_cases; #[cfg(test)] mod tests { @@ -219,13 +193,4 @@ mod tests { fn temp_app_paths_gives_each_call_its_own_root_even_for_one_label() { assert_each_call_gets_its_own_root(|| temp_app_paths("same-label").0); } - - #[test] - fn proptest_cases_are_raised_but_never_lowered() { - assert_eq!(raised_case_count(None, 256), 256); - assert_eq!(raised_case_count(Some("4096"), 256), 4096); - assert_eq!(raised_case_count(Some("16"), 256), 256); - assert_eq!(raised_case_count(Some("0"), 256), 256); - assert_eq!(raised_case_count(Some("lots"), 256), 256); - } } diff --git a/crates/rocm-core/tests/common/loopback.rs b/crates/rocm-core/tests/common/loopback.rs new file mode 100644 index 0000000000..40c05aa79f --- /dev/null +++ b/crates/rocm-core/tests/common/loopback.rs @@ -0,0 +1,108 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A loopback HTTP server for the integration tests that drive real network +//! code: it answers every request with one fixed `200 OK` until it is dropped. + +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::thread; +use std::time::Duration; + +/// Serves one fixed response on an ephemeral loopback port, to any number of +/// requests, until dropped. +/// +/// Dropping it never hangs: it wakes the accept loop with a connection of its +/// own before joining, so a client that never connected cannot leave the +/// server thread blocked in `accept`. +pub struct LoopbackServer { + port: u16, + stopping: Arc, + served: Arc, + handle: Option>, +} + +impl LoopbackServer { + pub fn start(content_type: Option<&str>, body: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + let port = listener.local_addr().expect("local addr").port(); + let stopping = Arc::new(AtomicBool::new(false)); + let served = Arc::new(AtomicUsize::new(0)); + let content_type = content_type + .map(|value| format!("Content-Type: {value}\r\n")) + .unwrap_or_default(); + let handle = thread::spawn({ + let stopping = Arc::clone(&stopping); + let served = Arc::clone(&served); + move || { + for stream in listener.incoming() { + if stopping.load(Ordering::SeqCst) { + break; + } + let Ok(mut stream) = stream else { continue }; + // A client that connects and never sends cannot hold the + // accept loop, and with it `Drop`, forever. + let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); + read_request_head(&mut stream); + // Counted before the response is written, so a client that + // has its answer always sees its request counted. + served.fetch_add(1, Ordering::SeqCst); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\n{content_type}Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(&body); + let _ = stream.flush(); + } + } + }); + Self { + port, + stopping, + served, + handle: Some(handle), + } + } + + pub fn url(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } + + /// Requests answered so far. A test that expects a verdict about the + /// response checks this is non-zero, so a request that never reached the + /// server cannot pass as one the code under test rejected. + pub fn requests_served(&self) -> usize { + self.served.load(Ordering::SeqCst) + } +} + +impl Drop for LoopbackServer { + fn drop(&mut self) { + self.stopping.store(true, Ordering::SeqCst); + // Wake the accept loop so it sees the flag. If even that connection + // fails, leave the thread detached rather than block on it. + let woken = TcpStream::connect(("127.0.0.1", self.port)).is_ok(); + if let Some(handle) = self.handle.take() + && woken + { + let _ = handle.join(); + } + } +} + +/// Read the request line and headers, so the response is not written into a +/// request still being sent. +fn read_request_head(stream: &mut TcpStream) { + let mut head = Vec::new(); + let mut chunk = [0_u8; 1024]; + while !head.windows(4).any(|window| window == b"\r\n\r\n") && head.len() < 64 * 1024 { + match stream.read(&mut chunk) { + Ok(0) | Err(_) => break, + Ok(read) => head.extend_from_slice(&chunk[..read]), + } + } +} diff --git a/crates/rocm-core/tests/common/proptest_cases.rs b/crates/rocm-core/tests/common/proptest_cases.rs new file mode 100644 index 0000000000..50b3090e5d --- /dev/null +++ b/crates/rocm-core/tests/common/proptest_cases.rs @@ -0,0 +1,54 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The case-count rule for property tests that set their own count. This file +//! is its one implementation: `rocm`'s and `rocm-core`'s crate-internal +//! `test_support`, the `rocm-core` and `rocm-engine-protocol` integration +//! tests, and the Lemonade engine's unit tests all include it by `#[path]`. +//! `archive_props::run_property` applies the same rule inline, since that file +//! is itself included by `#[path]` into three crates. + +/// Case count for a property test: `PROPTEST_CASES` (proptest's own knob, +/// which an explicit count would otherwise override) when it is set and above +/// `default`, otherwise `default`. +/// +/// Never below `default`: each count is chosen so its property reaches the +/// cases it targets, and some of those are hit by a small share of cases, so a +/// lower count would pass with the defect present. `PROPTEST_CASES=0` would +/// otherwise pass every property without running one. +pub fn proptest_cases(default: u32) -> u32 { + case_count_from_env("PROPTEST_CASES", default) +} + +/// Case count read from `var`, raising `default` but never lowering it, for +/// the same reason as [`proptest_cases`]. `rocm`'s uninstall properties use it +/// with their own knob, `ROCM_DELETION_PROP_CASES`. A value that is ignored, +/// because it is lower or does not parse, is reported on stderr. +pub fn case_count_from_env(var: &str, default: u32) -> u32 { + let requested = std::env::var(var).ok(); + let cases = raised_case_count(requested.as_deref(), default); + if let Some(requested) = requested + && requested.parse::().ok() != Some(cases) + { + eprintln!("{var}={requested} ignored: this property runs at least {default} cases"); + } + cases +} + +/// The pure half of [`case_count_from_env`]: `requested` raises `default` and +/// never lowers it. +pub fn raised_case_count(requested: Option<&str>, default: u32) -> u32 { + requested + .and_then(|value| value.parse().ok()) + .map_or(default, |requested: u32| requested.max(default)) +} + +#[test] +fn proptest_cases_can_raise_the_default_but_never_lower_it() { + assert_eq!(raised_case_count(None, 24), 24); + assert_eq!(raised_case_count(Some("1000"), 24), 1000); + assert_eq!(raised_case_count(Some("0"), 24), 24); + assert_eq!(raised_case_count(Some("8"), 24), 24); + assert_eq!(raised_case_count(Some("lots"), 24), 24); +} diff --git a/crates/rocm-core/tests/digest_gate_properties.rs b/crates/rocm-core/tests/digest_gate_properties.rs new file mode 100644 index 0000000000..c0aef3ca0f --- /dev/null +++ b/crates/rocm-core/tests/digest_gate_properties.rs @@ -0,0 +1,333 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for the digest gate in `download_file_streaming`, the single +//! place a downloaded artifact's bytes are authenticated. +//! +//! The generator is built around *near misses*, because that is the only region +//! where this gate can plausibly fail open: a uniformly random 64-character +//! string differs from the real digest in ~every position and is rejected by +//! any implementation, correct or not. What distinguishes a correct comparison +//! from a prefix comparison, a truncating comparison, or a trimming one is a +//! candidate that shares most of the real digest. + +#[path = "common/loopback.rs"] +mod loopback; +#[path = "common/proptest_cases.rs"] +mod proptest_cases; + +use std::time::Duration; + +use loopback::LoopbackServer; +use proptest::prelude::*; +use proptest_cases::proptest_cases; +use rocm_core::{DOWNLOAD_CHUNK_BYTES, DownloadRequest, download_file_streaming}; +use sha2::{Digest, Sha256}; + +/// A fresh scratch directory, removed when the returned guard drops — including +/// when an assertion fails, so a failing run leaves nothing in the temp dir. +fn scratch_dir(tag: &str) -> tempfile::TempDir { + tempfile::Builder::new() + .prefix(&format!("rocm-core-digest-gate-{tag}-")) + .tempdir() + .expect("scratch dir") +} + +/// Run one download against a loopback server, returning whether the gate +/// accepted the body. +fn download_accepts(body: &[u8], expected_sha256: Option<&str>, tag: &str) -> bool { + download_outcome(body, expected_sha256, tag).0.is_ok() +} + +/// Run one download against a loopback server, returning the download's +/// result and every file it left in its otherwise empty scratch directory — +/// the destination and any partial file, whatever it is named. +/// +/// Panics if the request never reached the server: a download that failed +/// before the body arrived proves nothing about the digest gate. +fn download_outcome( + body: &[u8], + expected_sha256: Option<&str>, + tag: &str, +) -> (Result<(), String>, Vec) { + let scratch = scratch_dir(tag); + let dir = scratch.path(); + let destination = dir.join("artifact.bin"); + let server = LoopbackServer::start(None, body.to_vec()); + let url = format!("{}/artifact.bin", server.url()); + let request = DownloadRequest { + expected_sha256, + ..DownloadRequest::new(&url, &destination, Duration::from_secs(10)) + }; + let result = download_file_streaming(&request) + .map(drop) + .map_err(|error| format!("{error:#}")); + assert!( + server.requests_served() > 0, + "the download never reached the test server: {result:?}" + ); + drop(server); + let mut left_behind: Vec = std::fs::read_dir(dir) + .expect("read scratch dir") + .map(|entry| { + entry + .expect("scratch entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + left_behind.sort(); + (result, left_behind) +} + +/// Whether `result` is the digest gate's rejection, rather than success or +/// some other failure. +fn is_digest_rejection(result: &Result<(), String>) -> bool { + result + .as_ref() + .is_err_and(|error| error.contains("SHA-256 mismatch")) +} + +fn digest_of(body: &[u8]) -> String { + format!("{:x}", Sha256::digest(body)) +} + +/// How to perturb the true digest. Each variant lands in the region where a +/// sloppy comparison would wrongly accept. +#[derive(Debug, Clone, Copy)] +enum NearMiss { + /// One hex nibble changed — the smallest possible difference. + OneNibble, + /// Correct prefix, wrong suffix: accepted by a prefix comparison. + WrongSuffix, + /// Correct prefix only, truncated: accepted by a `starts_with` comparison. + TruncatedPrefix, + /// Correct digest with a trailing newline, as a `.sha256` sidecar carries it. + TrailingNewline, + /// Correct digest with surrounding spaces. + Padded, + /// Correct digest with a 65th character appended. + Overlong, + /// Correct digest with a leading zero prepended (shifts every position). + LeadingZero, + /// Empty — the "no expectation" string, which must not read as a match. + Empty, + /// All-zero digest of the right shape. + AllZeroes, +} + +const ALL_NEAR_MISSES: [NearMiss; 9] = [ + NearMiss::OneNibble, + NearMiss::WrongSuffix, + NearMiss::TruncatedPrefix, + NearMiss::TrailingNewline, + NearMiss::Padded, + NearMiss::Overlong, + NearMiss::LeadingZero, + NearMiss::Empty, + NearMiss::AllZeroes, +]; + +fn apply_near_miss(digest: &str, miss: NearMiss, position: usize, keep: usize) -> String { + let position = position % digest.len(); + let keep = 1 + keep % (digest.len() - 1); + match miss { + NearMiss::OneNibble => { + let mut out: Vec = digest.chars().collect(); + out[position] = if out[position] == '0' { '1' } else { '0' }; + out.into_iter().collect() + } + NearMiss::WrongSuffix => { + let mut out = digest[..keep].to_owned(); + out.push_str(&"f".repeat(digest.len() - keep)); + if out == digest { + out.replace_range(keep..=keep, "0"); + } + out + } + NearMiss::TruncatedPrefix => digest[..keep].to_owned(), + NearMiss::TrailingNewline => format!("{digest}\n"), + NearMiss::Padded => format!(" {digest} "), + NearMiss::Overlong => format!("{digest}a"), + NearMiss::LeadingZero => format!("0{digest}"), + NearMiss::Empty => String::new(), + NearMiss::AllZeroes => "0".repeat(64), + } +} + +/// The positional near misses — one nibble changed, a correct prefix with a +/// wrong tail, a truncated prefix — at every position and every prefix length, +/// rather than at positions drawn at random: a regression that misses only one +/// position (a comparison that skips the last nibble, say) is caught on every +/// run, not on the runs that happen to draw it. +#[test] +fn positional_near_misses_are_rejected_at_every_position() { + let body = b"artifact body for the positional near misses".to_vec(); + let truth = digest_of(&body); + let mut not_rejected = Vec::new(); + for miss in [ + NearMiss::OneNibble, + NearMiss::WrongSuffix, + NearMiss::TruncatedPrefix, + ] { + for at in 0..truth.len() { + // `apply_near_miss` reads `position` for a changed nibble and + // `keep` (as `1 + keep % 63`) for a prefix length; `at` walks both. + let candidate = apply_near_miss(&truth, miss, at, at); + assert!( + !candidate.eq_ignore_ascii_case(&truth), + "near-miss kind {miss:?} at {at} reproduced the true digest" + ); + let (result, _) = download_outcome(&body, Some(&candidate), "positional"); + if !is_digest_rejection(&result) { + not_rejected.push(format!("{miss:?} at {at}: {candidate:?} gave {result:?}")); + } + } + } + assert!( + not_rejected.is_empty(), + "the digest gate did not reject these near misses of {truth:?}:\n{}", + not_rejected.join("\n") + ); +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(24), + ..ProptestConfig::default() + })] + + /// A near-miss digest must never be accepted, whatever the body. The one + /// deliberate exception is pure ASCII case folding, which the gate + /// implements as equivalent and which is covered separately below. + /// + /// Every kind of near miss is tried for every generated case, rather than + /// one kind drawn per case: a regression that admits just one kind would + /// otherwise go unnoticed in any run that happened not to draw it. The + /// positional kinds are also walked exhaustively by + /// [`positional_near_misses_are_rejected_at_every_position`]. + #[test] + fn near_miss_digests_are_rejected( + body in prop::collection::vec(any::(), 1..512), + position in any::(), + keep in any::(), + ) { + let truth = digest_of(&body); + for miss in ALL_NEAR_MISSES { + let candidate = apply_near_miss(&truth, miss, position, keep); + prop_assert!( + !candidate.eq_ignore_ascii_case(&truth), + "near-miss kind {miss:?} reproduced the true digest" + ); + let (result, _) = download_outcome(&body, Some(&candidate), "near-miss"); + prop_assert!( + is_digest_rejection(&result), + "the digest gate did not reject {candidate:?} for an artifact whose digest is \ + {truth:?} (near-miss kind {miss:?}): {result:?}" + ); + } + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(12), + ..ProptestConfig::default() + })] + + /// The true digest is accepted, in either ASCII case, and in mixed case — + /// case-insensitivity must hold in both directions, not just for a fully + /// lowercase expectation. + #[test] + fn the_true_digest_is_accepted_in_any_ascii_case( + body in prop::collection::vec(any::(), 1..512), + mask in prop::collection::vec(any::(), 64), + ) { + let truth = digest_of(&body); + let mixed: String = truth + .chars() + .zip(mask) + .map(|(ch, upper)| if upper { ch.to_ascii_uppercase() } else { ch }) + .collect(); + prop_assert!(download_accepts(&body, Some(&mixed), "mixed-case")); + } + + /// Mutating the *body* while keeping the expected digest must be rejected, + /// and nothing — no destination, no partial file — may be left behind for a + /// later run to treat as cached. This checks the end state: the download + /// cleans up on more than one path, and either one suffices here. + #[test] + fn a_tampered_body_is_rejected( + body in prop::collection::vec(any::(), 2..512), + index in any::(), + bit in 0u8..8, + ) { + let truth = digest_of(&body); + let mut tampered = body; + let position = index.index(tampered.len()); + tampered[position] ^= 1 << bit; + let (result, left_behind) = download_outcome(&tampered, Some(&truth), "tampered"); + prop_assert!( + is_digest_rejection(&result), + "a body differing by one bit was not rejected against the original digest: {result:?}" + ); + prop_assert!( + left_behind.is_empty(), + "a rejected download left {left_behind:?} behind" + ); + } +} + +/// A body several streaming chunks long, tampered only in its last byte, is +/// rejected and leaves nothing behind, while the intact body is accepted: the +/// digest must cover every byte, not just the first buffer. (A hash over only a +/// prefix is caught by the intact half; the tampered half catches a gate that +/// is skipped or disabled for a body this large.) +#[test] +fn tampering_past_the_first_chunk_is_rejected() { + // Three chunks plus a byte, so the body crosses a chunk boundary + // regardless of how short the socket reads come back. + let body: Vec = (0..=255_u8) + .cycle() + .take(3 * DOWNLOAD_CHUNK_BYTES + 1) + .collect(); + let truth = digest_of(&body); + assert!( + download_accepts(&body, Some(&truth), "multi-chunk-intact"), + "the untampered multi-chunk body must be accepted" + ); + let mut tampered = body; + *tampered.last_mut().expect("non-empty body") ^= 1; + let (result, left_behind) = download_outcome(&tampered, Some(&truth), "multi-chunk"); + assert!( + is_digest_rejection(&result), + "a change in the last byte was not detected: {result:?}" + ); + assert!( + left_behind.is_empty(), + "a rejected download left {left_behind:?} behind" + ); +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(8), + ..ProptestConfig::default() + })] + + /// Without an expected digest the download succeeds — this is the gate's + /// documented fail-open mode, pinned here so it cannot be mistaken for + /// verification. `DownloadOutcome::sha256` is reported but nothing compares + /// it, so a caller that omits `expected_sha256` has authenticated nothing. + #[test] + fn an_absent_expectation_authenticates_nothing( + body in prop::collection::vec(any::(), 1..256), + ) { + prop_assert!( + download_accepts(&body, None, "no-expectation"), + "callers rely on a digest-less download succeeding; it just proves nothing" + ); + } +} diff --git a/crates/rocm-core/tests/integrity_properties.rs b/crates/rocm-core/tests/integrity_properties.rs new file mode 100644 index 0000000000..eb782735ab --- /dev/null +++ b/crates/rocm-core/tests/integrity_properties.rs @@ -0,0 +1,256 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for the signature-verification path that decides whether a +//! downloaded artifact or metadata document is authentic. +//! +//! The generators here deliberately sample the *near-miss* region — a signature +//! with one byte changed, a signature re-encoded with leading zeros, a payload +//! with one bit flipped, a PEM mangled by Windows tooling — because a uniformly +//! random generator essentially never produces an input that reaches the +//! interesting branches of an RSA verifier. + +#[path = "common/proptest_cases.rs"] +mod proptest_cases; + +use std::sync::OnceLock; + +use proptest::prelude::*; +use proptest_cases::proptest_cases; +use rocm_core::{ + generate_rsa_signing_keypair, sign_rsa_pkcs1_sha256_signature, + verify_rsa_pkcs1_sha256_signature, +}; + +/// One 2048-bit keypair, generated once: key generation takes seconds in a debug build, and every +/// property below only needs *a* valid key, not a fresh one per case. +fn keypair() -> &'static (String, String) { + static KEYPAIR: OnceLock<(String, String)> = OnceLock::new(); + KEYPAIR.get_or_init(|| generate_rsa_signing_keypair().expect("keypair generation")) +} + +/// A second, unrelated keypair, for "signed by the wrong key" cases. +fn other_keypair() -> &'static (String, String) { + static KEYPAIR: OnceLock<(String, String)> = OnceLock::new(); + KEYPAIR.get_or_init(|| generate_rsa_signing_keypair().expect("keypair generation")) +} + +fn sign(payload: &[u8]) -> Vec { + sign_rsa_pkcs1_sha256_signature(&keypair().0, payload).expect("signing") +} + +fn verify(payload: &[u8], signature: &[u8]) -> bool { + verify_rsa_pkcs1_sha256_signature(&keypair().1, payload, signature, "test").is_ok() +} + +/// Payloads shaped like the documents actually verified here — arbitrary bytes, +/// near-empty bodies, and printable text such as a metadata JSON blob — capped +/// at 512 bytes so shrinking stays fast. +fn payload_strategy() -> impl Strategy> { + prop_oneof![ + 3 => prop::collection::vec(any::(), 0..512), + 1 => prop::collection::vec(any::(), 0..4), + 1 => "[ -~\\n]{0,400}".prop_map(String::into_bytes), + ] +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(64), + ..ProptestConfig::default() + })] + + /// Baseline: a genuine signature over the exact payload verifies. + #[test] + fn genuine_signature_verifies(payload in payload_strategy()) { + let signature = sign(&payload); + prop_assert!(verify(&payload, &signature)); + } + + /// Flipping a single bit of the payload must invalidate the signature. + /// This is the "tampered artifact" case: the attacker edits one byte of a + /// signed metadata document and keeps the original signature. + #[test] + fn single_bit_payload_mutation_is_rejected( + payload in prop::collection::vec(any::(), 1..256), + index in any::(), + bit in 0u8..8, + ) { + let signature = sign(&payload); + let mut tampered = payload.clone(); + let position = index.index(payload.len()); + tampered[position] ^= 1 << bit; + prop_assert!( + !verify(&tampered, &signature), + "a payload differing by one bit must not verify" + ); + } + + /// Truncating or extending the payload must invalidate the signature, even + /// when the retained prefix is byte-identical. + #[test] + fn payload_prefix_or_extension_is_rejected( + payload in prop::collection::vec(any::(), 2..256), + cut in any::(), + suffix in prop::collection::vec(any::(), 1..16), + ) { + let signature = sign(&payload); + let keep = cut.index(payload.len()); + prop_assert!(!verify(&payload[..keep], &signature)); + + let mut extended = payload; + extended.extend_from_slice(&suffix); + prop_assert!(!verify(&extended, &signature)); + } + + /// Mutating the signature — one byte changed, truncated, padded, or + /// re-encoded with leading zeros — must be rejected. Leading-zero + /// re-encoding is the classic PKCS#1 v1.5 pitfall: the integer value is + /// unchanged, so a verifier that parses to a bignum without length-checking + /// accepts it. + #[test] + fn signature_mutations_are_rejected( + payload in payload_strategy(), + index in any::(), + delta in 1u8..=255, + pad in 1usize..4, + ) { + let signature = sign(&payload); + + let mut flipped = signature.clone(); + let position = index.index(flipped.len()); + flipped[position] = flipped[position].wrapping_add(delta); + prop_assert!(!verify(&payload, &flipped), "a mutated signature must not verify"); + + prop_assert!(!verify(&payload, &signature[..signature.len() - 1])); + prop_assert!(!verify(&payload, &signature[1..])); + prop_assert!(!verify(&payload, &[])); + + let mut zero_padded = vec![0_u8; pad]; + zero_padded.extend_from_slice(&signature); + prop_assert!( + !verify(&payload, &zero_padded), + "a leading-zero re-encoding carries the same integer value and must \ + still be rejected on length" + ); + + let mut trailing = signature.clone(); + trailing.extend(std::iter::repeat_n(0_u8, pad)); + prop_assert!(!verify(&payload, &trailing)); + } + + /// A signature produced by a different key must never verify. + #[test] + fn signature_from_another_key_is_rejected(payload in payload_strategy()) { + let wrong = sign_rsa_pkcs1_sha256_signature(&other_keypair().0, &payload) + .expect("signing with the second key"); + prop_assert!(!verify(&payload, &wrong)); + } + + /// Arbitrary bytes offered as a signature must be rejected, never accepted + /// and never panic. + #[test] + fn arbitrary_bytes_are_never_a_valid_signature( + payload in payload_strategy(), + candidate in prop::collection::vec(any::(), 0..300), + ) { + prop_assert!(!verify(&payload, &candidate)); + } +} + +/// PEM mangling that real tooling produces: CRLF, lone CR, a UTF-8 BOM, a stray +/// trailing space, a blank line, a tab. Verification must still succeed for +/// each. The BOM, space, blank-line and tab styles depend on `normalize_pem`; +/// CRLF and lone CR are also accepted by the PEM parser itself, so those two +/// styles pin the outcome rather than `normalize_pem`'s handling of them. +fn mangle_pem(pem: &str, style: u8) -> String { + match style % 6 { + 0 => pem.replace('\n', "\r\n"), + 1 => pem.replace('\n', "\r"), + 2 => format!("\u{feff}{pem}"), + 3 => pem.replace('\n', " \n"), + 4 => pem.replace('\n', "\n\n"), + _ => pem.replace('\n', "\t\r\n"), + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(48), + ..ProptestConfig::default() + })] + + /// Tolerated mangling must not change *which* key is trusted. + #[test] + fn pem_mangling_preserves_the_verification_verdict( + payload in payload_strategy(), + style in any::(), + ) { + let signature = sign(&payload); + let mangled = mangle_pem(&keypair().1, style); + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&mangled, &payload, &signature, "test").is_ok(), + "whitespace/BOM mangling must not break a genuine signature" + ); + + // The same mangling applied to the *wrong* key must still reject. + let wrong = mangle_pem(&other_keypair().1, style); + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&wrong, &payload, &signature, "test").is_err(), + "normalization must not turn the wrong key into an accepting one" + ); + } + + /// Corrupting the base64 body of the public key must never yield a key that + /// accepts a signature made by the real key. + #[test] + fn corrupted_public_key_never_accepts( + payload in payload_strategy(), + index in any::(), + replacement in prop::sample::select( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + .chars() + .collect::>(), + ), + ) { + let signature = sign(&payload); + let pem = &keypair().1; + // Only the base64 body: everything after the header line and before + // the footer. + let body_start = pem.find('\n').expect("PEM header line") + 1; + let body_end = pem.find("-----END").expect("PEM footer"); + let body_positions: Vec = pem[body_start..body_end] + .char_indices() + .filter(|(_, ch)| ch.is_ascii_alphanumeric() || *ch == '+' || *ch == '/') + .map(|(position, _)| body_start + position) + .collect(); + let position = body_positions[index.index(body_positions.len())]; + let mut corrupted = pem.clone(); + corrupted.replace_range(position..=position, &replacement.to_string()); + if corrupted == *pem { + return Ok(()); + } + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&corrupted, &payload, &signature, "test").is_err(), + "a public key with a corrupted body must not accept the real signature" + ); + } + + /// Garbage offered as a public key must produce an error, never a success + /// and never a panic. + #[test] + fn arbitrary_public_key_text_is_rejected( + payload in payload_strategy(), + pem in prop_oneof![ + Just(String::new()), + "[ -~\\n]{0,200}", + "-----BEGIN PUBLIC KEY-----\n[A-Za-z0-9+/=\\n]{0,200}\n-----END PUBLIC KEY-----\n", + ], + ) { + let signature = sign(&payload); + prop_assert!( + verify_rsa_pkcs1_sha256_signature(&pem, &payload, &signature, "test").is_err() + ); + } +} diff --git a/crates/rocm-core/tests/model_selector_properties.rs b/crates/rocm-core/tests/model_selector_properties.rs new file mode 100644 index 0000000000..75b56089eb --- /dev/null +++ b/crates/rocm-core/tests/model_selector_properties.rs @@ -0,0 +1,626 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for model selection and for the readiness decision the CLI +//! makes from an engine's `/v1/models` response. +//! +//! Two separate risks live here. Selection must be *deterministic and +//! unambiguous*: `resolve_model_recipe` takes the first recipe whose +//! `matches_ref` returns true, so two recipes answering to the same selector +//! would make the winner an accident of catalog order. And readiness is decided +//! from a JSON document served by the engine — untrusted input — so it must +//! never panic and must not report a *different* model as the one the user +//! asked for. + +#[path = "common/loopback.rs"] +mod loopback; +#[path = "common/proptest_cases.rs"] +mod proptest_cases; + +use std::time::Duration; + +use loopback::LoopbackServer; +use proptest::prelude::*; +use proptest_cases::proptest_cases; +use rocm_core::{ModelRecipeRecord, builtin_model_recipes, openai_models_endpoint_has_model}; + +fn catalog() -> &'static [ModelRecipeRecord] { + use std::sync::OnceLock; + static CATALOG: OnceLock> = OnceLock::new(); + CATALOG.get_or_init(builtin_model_recipes) +} + +/// Every selector the shipped catalog advertises: canonical ids and aliases. +fn advertised_selectors() -> Vec<(String, String)> { + let mut selectors = Vec::new(); + for recipe in catalog() { + selectors.push(( + recipe.canonical_model_id.clone(), + recipe.canonical_model_id.clone(), + )); + for alias in &recipe.aliases { + selectors.push((alias.clone(), recipe.canonical_model_id.clone())); + } + } + selectors +} + +/// A `/v1/models` document listing exactly `ids`. +fn models_document(ids: &[&str]) -> String { + serde_json::json!({ + "object": "list", + "data": ids + .iter() + .map(|id| serde_json::json!({ "id": id, "object": "model" })) + .collect::>(), + }) + .to_string() +} + +/// Ask the readiness check what it makes of the document `server` serves when +/// expecting `expected`. Panics if the request never reached the server, so a +/// transport failure cannot pass as a "not ready" verdict. +fn readiness_from(server: &LoopbackServer, expected: Option<&str>) -> Result { + let before = server.requests_served(); + let outcome = + openai_models_endpoint_has_model(&server.url(), expected, None, Duration::from_secs(10)) + .map_err(|error| error.to_string()); + assert!( + server.requests_served() > before, + "the readiness check never reached the test server: {outcome:?}" + ); + outcome +} + +/// Ask the readiness check what it makes of `body` when expecting `expected`. +fn endpoint_reports(body: String, expected: Option<&str>) -> Result { + let server = LoopbackServer::start(Some("application/json"), body.into_bytes()); + readiness_from(&server, expected) +} + +#[test] +fn the_shipped_catalog_is_not_empty() { + assert!( + !catalog().is_empty(), + "these properties are vacuous without a catalog to check" + ); +} + +/// Every advertised selector must resolve to the recipe that advertises it. +/// +/// That no selector is advertised twice is the catalog schema's own check +/// (`ModelRecipeIndexDocument::validate`, which normalizes refs the same way +/// `matches_ref` does), run on the shipped catalog by its unit tests. +#[test] +fn every_advertised_selector_resolves_to_its_own_recipe() { + for (selector, owner) in advertised_selectors() { + let resolved: Vec<&str> = catalog() + .iter() + .filter(|recipe| recipe.matches_ref(&selector)) + .map(|recipe| recipe.canonical_model_id.as_str()) + .collect(); + assert_eq!( + resolved, + vec![owner.as_str()], + "selector {selector:?} must resolve to exactly its own recipe" + ); + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(512), + ..ProptestConfig::default() + })] + + /// Selector matching ignores surrounding whitespace and ASCII case: any + /// re-casing of an advertised selector, padded with spaces, tabs or + /// newlines, still selects exactly its own recipe. (That other edits do + /// not match is the next property's job.) + #[test] + fn selector_matching_tolerates_only_case_and_surrounding_whitespace( + index in any::(), + upper in prop::collection::vec(any::(), 0..80), + lead in "[ \\t\\n]{0,3}", + trail in "[ \\t\\n]{0,3}", + ) { + let selectors = advertised_selectors(); + let (selector, owner) = &selectors[index.index(selectors.len())]; + let recased: String = selector + .chars() + .enumerate() + .map(|(position, ch)| { + if upper.get(position).copied().unwrap_or(false) { + ch.to_ascii_uppercase() + } else { + ch + } + }) + .collect(); + let decorated = format!("{lead}{recased}{trail}"); + let resolved: Vec<&str> = catalog() + .iter() + .filter(|recipe| recipe.matches_ref(&decorated)) + .map(|recipe| recipe.canonical_model_id.as_str()) + .collect(); + prop_assert_eq!( + resolved, + vec![owner.as_str()], + "case/whitespace decoration of {:?} changed which recipe it selects", + selector + ); + } + + /// Selection is otherwise exact: an advertised selector with one character + /// inserted (interior whitespace included) or deleted selects no recipe at + /// all — neither its own (no fuzzy matching) nor another — unless the edit + /// happens to spell some other advertised selector, up to ASCII case. + #[test] + fn a_mistyped_selector_selects_nothing( + index in any::(), + position in any::(), + delete in any::(), + inserted in prop::sample::select( + "abcdefghijklmnopqrstuvwxyz0123456789-_./ \t".chars().collect::>() + ), + ) { + let selectors = advertised_selectors(); + let (selector, _) = &selectors[index.index(selectors.len())]; + let mut mistyped: Vec = selector.chars().collect(); + if delete { + let at = position.index(mistyped.len()); + mistyped.remove(at); + } else { + let at = position.index(mistyped.len() + 1); + // Whitespace at either end is surrounding whitespace, which the + // previous property covers and selection deliberately ignores. + prop_assume!( + !(inserted.is_whitespace() && (at == 0 || at == mistyped.len())) + ); + mistyped.insert(at, inserted); + } + let mistyped: String = mistyped.into_iter().collect(); + prop_assume!( + !selectors + .iter() + .any(|(other, _)| other.eq_ignore_ascii_case(&mistyped)) + ); + + let selected: Vec<&str> = catalog() + .iter() + .filter(|recipe| recipe.matches_ref(&mistyped)) + .map(|recipe| recipe.canonical_model_id.as_str()) + .collect(); + prop_assert!( + selected.is_empty(), + "mistyped selector {:?} (from {:?}) selected {:?}", + mistyped, + selector, + selected + ); + } +} + +/// `/v1/models` bodies that name no model matching any of the expectations +/// below, shaped the way a real engine shapes them, plus the degenerate shapes +/// a broken or hostile one could send. +/// +/// The listed ids carry no letter. Every expected name below does, and every +/// rule readiness matches by — equality, basename, catalog alias, family word — +/// needs a letter in common, so none of these documents may read as ready. +fn models_body_strategy() -> impl Strategy { + prop_oneof![ + 3 => prop::collection::vec("[0-9 ._:#/@-]{0,24}", 0..4).prop_map(|ids| { + let ids: Vec<&str> = ids.iter().map(String::as_str).collect(); + models_document(&ids) + }), + 1 => Just("{}".to_owned()), + 1 => Just(r#"{"data": []}"#.to_owned()), + 1 => Just(r#"{"data": null}"#.to_owned()), + 1 => Just(r#"{"data": [{}]}"#.to_owned()), + 1 => Just(r#"{"data": [{"id": null}]}"#.to_owned()), + 1 => Just(r#"{"data": [[]]}"#.to_owned()), + 1 => Just(r#"{"data": "not-an-array"}"#.to_owned()), + 1 => Just("[]".to_owned()), + 1 => Just("null".to_owned()), + 1 => "[ -~]{0,64}", + ] +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(48), + ..ProptestConfig::default() + })] + + /// A `/v1/models` document that names no matching model — malformed, + /// hostile, or listing only unrelated ids — must produce a clean `false` or + /// a clean error: never a panic, and never "ready". A check that reported + /// ready for any non-empty listing, ignoring the expected model, fails here. + #[test] + fn a_models_document_naming_no_matching_model_never_reports_ready( + body in models_body_strategy(), + expected in prop::sample::select(vec![ + "Qwen/Qwen3-4B-Instruct-2507", + "meta-llama/Llama-3.1-8B-Instruct", + "nonexistent/model", + ]), + ) { + let verdict = endpoint_reports(body.clone(), Some(expected)); + prop_assert_ne!( + verdict, + Ok(true), + "reported ready for {:?} from a document naming no matching model: {}", + expected, + body + ); + } + + /// Positive control for the property above: a document that does list the + /// expected model — exactly, re-cased, padded, or by its basename alone, as + /// engines commonly report it — among other entries reads as ready. Without + /// this, a check that never reported ready would pass every negative + /// property here. + /// It pins the outcome, not which matching rule produces it: several rules + /// accept these spellings, so it does not stand guard over any one of them. + #[test] + fn a_models_document_listing_the_expected_model_reports_ready( + others in prop::collection::vec("[ -~]{0,24}", 0..3), + at in any::(), + spelling in 0u8..4, + expected in prop::sample::select(vec![ + "Qwen/Qwen3-4B-Instruct-2507", + "meta-llama/Llama-3.1-8B-Instruct", + "nonexistent/model", + ]), + ) { + let listed = match spelling { + 0 => expected.to_owned(), + 1 => expected.to_ascii_uppercase(), + 2 => format!(" {expected}\t"), + _ => expected.rsplit('/').next().unwrap_or(expected).to_owned(), + }; + let mut ids = others; + let position = at.index(ids.len() + 1); + ids.insert(position, listed.clone()); + let ids: Vec<&str> = ids.iter().map(String::as_str).collect(); + let body = models_document(&ids); + prop_assert_eq!( + endpoint_reports(body, Some(expected)), + Ok(true), + "{:?} listed as {:?} was not reported ready", + expected, + listed + ); + } +} + +/// Whether `name` is a bare *family* word — a single alphabetic token of three +/// or more characters with no digit, like `qwen`, `llama` or `gemma`. +/// +/// Approximates the shape `rocm_core`'s private family matcher treats as a +/// family: readiness matches such a word against any model with a name token +/// that starts with it, so these names are excluded from the strict property +/// below and pinned separately. (Being an approximation, it only decides which +/// catalog names that property skips; the near-miss names that property also +/// checks never pass through it.) +fn is_bare_family_word(name: &str) -> bool { + let normalized: String = name + .trim() + .chars() + .filter(char::is_ascii_alphanumeric) + .flat_map(char::to_lowercase) + .collect(); + let single_token = name + .trim() + .split(|ch: char| !ch.is_ascii_alphanumeric()) + .filter(|token| !token.is_empty()) + .count() + == 1; + single_token && normalized.len() >= 3 && !normalized.chars().any(|ch| ch.is_ascii_digit()) +} + +/// An engine that lists only a *different* catalog model must not satisfy a +/// request for the one the user asked for. This is the "silently serving an +/// unrelated model" case: readiness is what gates the traffic the user then +/// sends, so a false positive routes inference to the wrong weights. +/// +/// Checked exhaustively over every ordered pair of catalog entries — including +/// each entry's aliases as the served id, since an engine echoes back whatever +/// name it was launched with — rather than sampled. The catalog is small, the +/// confusable pairs in it are specific (several `Qwen*` variants differing only +/// in size and quantization), and a sampling strategy can miss exactly those. +/// +/// Each wanted model is also checked against near misses of its own name that +/// it does not answer to: the name with a quantization or variant suffix, the +/// first name token containing a digit (`qwen3`, or just `3` for a Llama), a +/// two-letter stub, and the leading word with its first letter dropped +/// (`wen`). Those are what separate an exact comparison from a substring one, +/// and a family match from one that has lost its guards or matches inside a +/// word rather than at its start. +/// +/// Two shapes that *do* cross-match today are excluded here and pinned +/// separately, so they stay visible: bare family words, by +/// [`a_bare_family_word_satisfies_readiness_for_the_whole_family`], and one +/// recipe's alias spelling another recipe's basename, by +/// [`an_alias_spelling_another_models_basename_reads_as_ready_for_it`]. +/// +/// The request count grows with the square of the catalog (every served name +/// against every wanted model), so each served name gets one loopback server +/// that answers all of its requests, rather than a server per request. +#[test] +fn listing_a_different_catalog_model_is_never_readiness() { + let recipes = catalog(); + assert!(recipes.len() > 1, "need at least two recipes to confuse"); + let mut false_positives = Vec::new(); + + for served in recipes { + let served_names = std::iter::once(&served.canonical_model_id).chain(&served.aliases); + for served_name in served_names { + if is_bare_family_word(served_name) { + continue; + } + let server = LoopbackServer::start( + Some("application/json"), + models_document(&[served_name.as_str()]).into_bytes(), + ); + for wanted in recipes { + // Skip the served recipe itself, a name the *wanted* recipe + // also answers to (a genuine shared alias, not a confusion), + // and the pinned alias/basename collision. + if wanted.canonical_model_id == served.canonical_model_id + || wanted.matches_ref(served_name) + || spells_the_basename_of(served_name, wanted) + { + continue; + } + if readiness_from(&server, Some(&wanted.canonical_model_id)) == Ok(true) { + false_positives.push(format!( + "wanted {:?} but an endpoint serving only {:?} reported ready", + wanted.canonical_model_id, served_name + )); + } + } + } + } + + for wanted in recipes { + for served_name in near_miss_names(&wanted.canonical_model_id) { + if wanted.matches_ref(&served_name) { + continue; + } + let body = models_document(&[served_name.as_str()]); + if endpoint_reports(body, Some(&wanted.canonical_model_id)) == Ok(true) { + false_positives.push(format!( + "wanted {:?} but an endpoint serving only the near miss {:?} reported ready", + wanted.canonical_model_id, served_name + )); + } + } + } + + assert!( + false_positives.is_empty(), + "readiness matched an unrelated model:\n{}", + false_positives.join("\n") + ); +} + +/// Whether `name`, as an engine would report it, is exactly the basename of +/// `recipe`'s canonical id (up to ASCII case) — the part after the last `/`, +/// which readiness accepts on its own because engines commonly report a model +/// by its basename. +fn spells_the_basename_of(name: &str, recipe: &ModelRecipeRecord) -> bool { + let basename = recipe + .canonical_model_id + .rsplit('/') + .next() + .unwrap_or(&recipe.canonical_model_id); + name.trim().eq_ignore_ascii_case(basename) +} + +/// Characterization test for a known false-positive surface, pinned so it is +/// visible rather than merely absent from the suite. +/// +/// Readiness accepts a model's basename for the model (an engine launched on +/// `Qwen/Qwen3-32B` commonly reports `Qwen3-32B`). Some catalog aliases spell +/// the basename of a *different* catalog model: `qwen3-32b` is an alias of +/// `Qwen/Qwen3-32B-FP8` and also the basename of `Qwen/Qwen3-32B`. An engine +/// launched on the FP8 recipe under that alias reports `qwen3-32b`, and that +/// reads as ready for `Qwen/Qwen3-32B` — readiness cannot tell the two apart +/// from the name alone. +/// +/// The collisions are derived from the catalog and compared with the list +/// below, so a catalog change that adds one fails here and has to be looked +/// at rather than being silently excluded from +/// [`listing_a_different_catalog_model_is_never_readiness`]. If readiness or +/// the catalog aliases are changed to remove a collision, update the list +/// along with them. +#[test] +fn an_alias_spelling_another_models_basename_reads_as_ready_for_it() { + let mut collisions = Vec::new(); + for owner in catalog() { + for alias in &owner.aliases { + for other in catalog() { + if other.canonical_model_id != owner.canonical_model_id + && !other.matches_ref(alias) + && spells_the_basename_of(alias, other) + { + collisions.push(( + alias.as_str(), + owner.canonical_model_id.as_str(), + other.canonical_model_id.as_str(), + )); + } + } + } + } + collisions.sort_unstable(); + assert_eq!( + collisions, + vec![ + ("qwen3-32b", "Qwen/Qwen3-32B-FP8", "Qwen/Qwen3-32B"), + ( + "qwen3.6-35b-a3b", + "unsloth/Qwen3.6-35B-A3B-GGUF:Q4_K_M", + "Qwen/Qwen3.6-35B-A3B" + ), + ], + "(alias, recipe it belongs to, recipe whose basename it spells) — the catalog's \ + alias/basename collisions changed" + ); + + for (alias, owner, other) in collisions { + assert_eq!( + endpoint_reports(models_document(&[alias]), Some(other)), + Ok(true), + "pinning the current behavior: {owner:?} served as {alias:?} reads as ready for \ + {other:?}" + ); + } +} + +/// Readiness accepts a catalog model under any of its names: an engine that +/// reports an alias satisfies a request for the canonical id, and the reverse. +/// Checked for every alias in the shipped catalog. +#[test] +fn readiness_accepts_a_catalog_model_under_any_of_its_names() { + let mut misses = Vec::new(); + for recipe in catalog() { + for alias in &recipe.aliases { + for (served, wanted) in [ + (alias.as_str(), recipe.canonical_model_id.as_str()), + (recipe.canonical_model_id.as_str(), alias.as_str()), + ] { + let body = models_document(&[served]); + if endpoint_reports(body, Some(wanted)) != Ok(true) { + misses.push(format!("serving {served:?} did not satisfy {wanted:?}")); + } + } + } + } + assert!(misses.is_empty(), "{}", misses.join("\n")); +} + +/// Names that sit next to `id` without being it: `id` with a variant suffix, +/// the first token of its basename that contains a digit, a two-letter stub of +/// its basename, and the basename's leading letters minus the first one. +fn near_miss_names(id: &str) -> Vec { + let basename = id.rsplit('/').next().unwrap_or(id); + let mut names: Vec = ["-FP8", "-GGUF", "-Instruct", "-v2"] + .iter() + .map(|suffix| format!("{id}{suffix}")) + .collect(); + if let Some(first_token) = basename + .split(|ch: char| !ch.is_ascii_alphanumeric()) + .find(|token| token.chars().any(|ch| ch.is_ascii_digit())) + { + names.push(first_token.to_ascii_lowercase()); + } + let stub: String = basename + .chars() + .filter(char::is_ascii_alphabetic) + .take(2) + .collect(); + if stub.len() == 2 { + names.push(stub.to_ascii_lowercase()); + } + let leading_word: String = basename + .chars() + .take_while(char::is_ascii_alphabetic) + .collect(); + if leading_word.len() >= 4 { + names.push(leading_word[1..].to_ascii_lowercase()); + } + names +} + +/// Characterization test for a known false-positive surface, pinned so it is +/// visible rather than merely absent from the suite. +/// +/// Readiness matches a bare *family* word against any model with a name token +/// that starts with it, and does so in both directions. One direction is +/// exercised by an existing unit test in `endpoint_readiness.rs`: asking for `qwen` is +/// satisfied by an engine serving `Qwen3-0.6B-GGUF`. This test pins the other: an endpoint +/// whose `/v1/models` reports only the bare id `qwen` — itself a catalog alias +/// — is reported ready for other `Qwen*` entries in the catalog, not just the +/// one it actually loaded. The test asserts that such cross-matches exist and +/// that they stay inside the family. +/// +/// So when an engine reports nothing more specific than the family word, +/// readiness cannot tell the models of that family apart: a request for one +/// `Qwen*` model reads as ready whichever `Qwen*` model the engine has +/// resident. Note the asymmetry — *selection* (`matches_ref`) is exact up to +/// case and whitespace, while the readiness check that confirms the engine is +/// serving the selected model is looser. +/// +/// If readiness is ever tightened, this test is expected to fail and should be +/// updated along with the behavior. +#[test] +fn a_bare_family_word_satisfies_readiness_for_the_whole_family() { + let body = models_document(&["qwen"]); + + let cross_matched: Vec<&str> = catalog() + .iter() + .map(|recipe| recipe.canonical_model_id.as_str()) + // Every catalog entry except the one that genuinely owns the `qwen` + // alias. + .filter(|id| { + !catalog() + .iter() + .find(|recipe| recipe.canonical_model_id == *id) + .is_some_and(|recipe| recipe.matches_ref("qwen")) + }) + .filter(|id| endpoint_reports(body.clone(), Some(id)) == Ok(true)) + .collect(); + + assert!( + !cross_matched.is_empty(), + "expected the documented family-word cross-match to still be present" + ); + assert!( + cross_matched + .iter() + .all(|id| id.to_ascii_lowercase().contains("qwen")), + "the cross-match must at least stay inside the family: {cross_matched:?}" + ); + + // The other direction, which the `endpoint_readiness.rs` unit test also exercises: asking + // for the bare family word is satisfied by an engine serving any catalog + // model of that family. + for recipe in catalog().iter().filter(|recipe| { + recipe + .canonical_model_id + .to_ascii_lowercase() + .contains("qwen") + }) { + let body = models_document(&[recipe.canonical_model_id.as_str()]); + assert_eq!( + endpoint_reports(body, Some("qwen")), + Ok(true), + "asking for `qwen` was not satisfied by a served {:?}", + recipe.canonical_model_id + ); + } +} + +/// The same behavior outside the catalog: a user can `rocm serve` any Hugging +/// Face ref, and readiness compares the engine's reported id against it. A bare +/// family word therefore satisfies a request for any model of that family, +/// whatever its size. +#[test] +fn a_bare_family_word_satisfies_readiness_for_an_uncatalogued_model() { + for (served, wanted) in [ + ("qwen", "Qwen/Qwen3-235B-A22B-Instruct"), + ("llama", "meta-llama/Llama-3.1-70B-Instruct"), + ("gemma", "google/gemma-2-27b-it"), + ] { + let body = models_document(&[served]); + assert_eq!( + endpoint_reports(body, Some(wanted)), + Ok(true), + "pinning the current behavior: serving {served:?} reads as ready for {wanted:?}" + ); + } +} diff --git a/crates/rocm-engine-protocol/Cargo.toml b/crates/rocm-engine-protocol/Cargo.toml index 19bd5f483a..1d60c6db3e 100644 --- a/crates/rocm-engine-protocol/Cargo.toml +++ b/crates/rocm-engine-protocol/Cargo.toml @@ -14,3 +14,7 @@ workspace = true rocm-core = { path = "../rocm-core" } serde.workspace = true serde_json.workspace = true + +[dev-dependencies] +# Property tests for the engine protocol decoders (`tests/protocol_properties.rs`). +proptest = { version = "1", default-features = false, features = ["std"] } diff --git a/crates/rocm-engine-protocol/tests/protocol_properties.rs b/crates/rocm-engine-protocol/tests/protocol_properties.rs new file mode 100644 index 0000000000..8aa54b9971 --- /dev/null +++ b/crates/rocm-engine-protocol/tests/protocol_properties.rs @@ -0,0 +1,1033 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Property tests for the engine protocol: the types a plugin's stdout is +//! decoded into, and the helpers that turn untrusted filesystem and CLI input +//! into identifiers the CLI then acts on. +//! +//! Decoding of arbitrary bytes, arbitrary text and byte-level mutations of +//! valid encodings is smoke-tested for panics only: every type here uses +//! derived `Deserialize`, so those tests guard against a future hand-written +//! decoder or an unwrap on the decode path, not against wrong values. What a +//! decode must produce is pinned by the round-trip, version-skew and golden +//! document tests. + +#[path = "../../rocm-core/tests/common/proptest_cases.rs"] +mod proptest_cases; + +use proptest::prelude::*; +use proptest_cases::proptest_cases; +use rocm_engine_protocol::{ + DetectResponse, DevicePolicy, EngineCapabilities, EngineDeviceAvailability, EngineMethod, + EngineRecipeEndpointHint, EngineRecipeHint, EngineRecipeUnsupportedCombinationHint, + EngineRequestEnvelope, EngineResponseEnvelope, GpuSelection, HealthcheckResponse, + LaunchRequest, LogsRequest, LogsResponse, ResolveModelRequest, + engine_id_from_plugin_binary_name, gpu_indices_to_csv, is_public_bind_host, launch_gpu_indices, +}; +use std::collections::BTreeMap; + +/// Decode `bytes` as `T`, asserting only that it does not panic. +macro_rules! decode_without_panic { + ($type:ty, $bytes:expr) => { + let _ = serde_json::from_slice::<$type>($bytes); + }; +} + +/// Every envelope and payload type the protocol decodes, in either direction: +/// what a plugin writes (responses) and what it reads (requests). +fn decode_all_without_panic(bytes: &[u8]) { + decode_without_panic!(EngineRequestEnvelope, bytes); + decode_without_panic!(EngineResponseEnvelope, bytes); + decode_without_panic!(DetectResponse, bytes); + decode_without_panic!(HealthcheckResponse, bytes); + decode_without_panic!(LaunchRequest, bytes); + decode_without_panic!(LogsRequest, bytes); + decode_without_panic!(LogsResponse, bytes); + decode_without_panic!(ResolveModelRequest, bytes); + decode_without_panic!(EngineRecipeHint, bytes); + decode_without_panic!(GpuSelection, bytes); +} + +/// `any::()` deliberately excludes NaN and the infinities, so a naive +/// generator never samples the values JSON has no representation for — exactly +/// the region where a float round trip can break. Sample them explicitly. +fn tokens_per_sec_strategy() -> impl Strategy> { + proptest::option::of(prop_oneof![ + 6 => any::(), + 1 => Just(f32::NAN), + 1 => Just(f32::INFINITY), + 1 => Just(f32::NEG_INFINITY), + 1 => Just(f32::MIN_POSITIVE), + 1 => Just(-0.0_f32), + ]) +} + +fn healthcheck_strategy() -> impl Strategy { + ( + "[ -~]{0,20}", + any::(), + "[ -~]{0,20}", + any::(), + any::(), + proptest::option::of("[ -~]{0,30}"), + tokens_per_sec_strategy(), + ) + .prop_map( + |( + status, + model_loaded, + device, + uptime_sec, + queue_depth, + last_error, + tokens_per_sec, + )| { + HealthcheckResponse { + status, + model_loaded, + device, + uptime_sec, + queue_depth, + last_error, + tokens_per_sec, + } + }, + ) +} + +fn recipe_hint_strategy() -> impl Strategy { + ( + "[ -~]{0,10}", + "[ -~]{0,10}", + prop::collection::vec("[ -~]{0,12}", 0..4), + prop::collection::btree_map("[ -~]{0,8}", "[ -~]{0,8}", 0..4), + // Often empty: an empty map is omitted on the wire, so both the + // omitted and the present spelling have to round-trip. + prop::collection::btree_map("[ -~]{0,8}", "[ -~]{0,8}", 0..3), + proptest::option::of(( + "[ -~]{0,8}", + prop::collection::btree_map("[ -~]{0,8}", "[ -~]{0,8}", 0..3), + )), + prop::collection::vec(("[ -~]{0,8}", "[ -~]{0,8}"), 0..3), + prop::collection::vec("[ -~]{0,8}", 0..3), + ) + .prop_map( + |( + contract_version, + engine, + required_flags, + parser_settings, + extra_env, + preferred_endpoint, + unsupported, + notes, + )| EngineRecipeHint { + contract_version, + engine, + required_flags, + parser_settings, + extra_env, + preferred_endpoint: preferred_endpoint.map(|(endpoint_mode, settings)| { + EngineRecipeEndpointHint { + endpoint_mode, + settings, + } + }), + unsupported_combinations: unsupported + .into_iter() + .map( + |(combination, reason)| EngineRecipeUnsupportedCombinationHint { + combination, + reason, + }, + ) + .collect(), + notes, + }, + ) +} + +fn optional_text() -> impl Strategy> { + proptest::option::of("[ -~]{0,10}") +} + +fn detect_response_strategy() -> impl Strategy { + ( + ( + any::(), + optional_text(), + optional_text(), + optional_text(), + proptest::option::of(any::()), + optional_text(), + optional_text(), + optional_text(), + ), + prop::collection::vec( + ( + "[ -~]{0,8}", + any::(), + proptest::option::of("[ -~]{0,8}"), + ), + 0..4, + ), + ( + any::<[bool; 5]>(), + "[ -~]{0,8}", + prop::collection::vec("[ -~]{0,8}", 0..4), + ), + ) + .prop_map( + |( + ( + installed, + env_id, + runtime_kind, + runtime_executable, + managed_env, + python_version, + torch_version, + transformers_version, + ), + devices, + ( + [ + cpu, + rocm_gpu, + openai_compatible, + tool_calling, + reasoning_parser, + ], + quantized_models, + notes, + ), + )| DetectResponse { + installed, + env_id, + runtime_kind, + runtime_executable, + managed_env, + python_version, + torch_version, + transformers_version, + available_devices: devices + .into_iter() + .map(|(kind, available, reason)| EngineDeviceAvailability { + kind, + available, + reason, + }) + .collect(), + capabilities: EngineCapabilities { + cpu, + rocm_gpu, + openai_compatible, + tool_calling, + quantized_models, + reasoning_parser, + }, + notes, + }, + ) +} + +/// Byte-level mutations of a valid encoding: the only generator that reliably +/// reaches field-level decode logic. +fn mutate(json: &str, position: usize, kind: u8) -> Vec { + let mut bytes = json.as_bytes().to_vec(); + if bytes.is_empty() { + return bytes; + } + let position = position % bytes.len(); + match kind % 6 { + 0 => bytes[position] = b'"', + 1 => bytes[position] = b'{', + 2 => bytes[position] = 0xff, + 3 => { + bytes.truncate(position); + } + 4 => { + bytes.remove(position); + } + _ => bytes.insert(position, b'\\'), + } + bytes +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(512), + ..ProptestConfig::default() + })] + + /// Encode then decode must be the identity for every value the type holds. + #[test] + fn healthcheck_response_roundtrips(response in healthcheck_strategy()) { + let encoded = serde_json::to_string(&response).expect("encode"); + let decoded: HealthcheckResponse = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(decoded.status, response.status); + prop_assert_eq!(decoded.model_loaded, response.model_loaded); + prop_assert_eq!(decoded.device, response.device); + prop_assert_eq!(decoded.uptime_sec, response.uptime_sec); + prop_assert_eq!(decoded.queue_depth, response.queue_depth); + prop_assert_eq!(decoded.last_error, response.last_error); + // JSON has no representation for NaN or the infinities, so `serde_json` + // encodes them as `null` and they come back as `None`. Every *finite* + // value must round-trip exactly; a non-finite one is documented here as + // collapsing to "unknown" rather than surviving or erroring. + match response.tokens_per_sec { + Some(value) if !value.is_finite() => prop_assert_eq!( + decoded.tokens_per_sec, + None, + "a non-finite rate has no JSON representation and reads back as unknown" + ), + expected => prop_assert_eq!( + decoded.tokens_per_sec.map(f32::to_bits), + expected.map(f32::to_bits), + "a finite tokens_per_sec must survive an encode/decode round trip" + ), + } + } + + #[test] + fn recipe_hint_roundtrips(hint in recipe_hint_strategy()) { + let encoded = serde_json::to_string(&hint).expect("encode"); + let decoded: EngineRecipeHint = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(&decoded, &hint); + // Duplicate-serve reuse compares a recipe hint's JSON as a string, so an + // empty `extra_env` must stay off the wire: a hint encodes exactly as it + // did before the field existed. + let value = serde_json::to_value(&hint).expect("encode"); + prop_assert_eq!( + value.get("extra_env").is_some(), + !hint.extra_env.is_empty(), + "extra_env must be omitted exactly when it is empty" + ); + } + + #[test] + fn detect_response_roundtrips(response in detect_response_strategy()) { + let encoded = serde_json::to_string(&response).expect("encode"); + let decoded: DetectResponse = serde_json::from_str(&encoded).expect("decode"); + // `DetectResponse` has no `PartialEq`; compare the whole struct through + // its JSON value instead of a hand-picked subset of fields. (Being a + // round trip, this cannot see a renamed field.) + prop_assert_eq!( + serde_json::to_value(&decoded).expect("re-encode"), + serde_json::to_value(&response).expect("encode") + ); + } + + /// `EngineResponseEnvelope::success` serializes caller data with `expect`; + /// it must not panic for the response types the engines actually send, and + /// the data must arrive intact on the other side of the wire. + #[test] + fn response_envelope_success_carries_the_data_intact(response in healthcheck_strategy()) { + let envelope = EngineResponseEnvelope::success(&response); + prop_assert!(envelope.ok); + let encoded = serde_json::to_string(&envelope).expect("encode"); + let decoded: EngineResponseEnvelope = serde_json::from_str(&encoded).expect("decode"); + prop_assert!(decoded.ok); + prop_assert!(decoded.error.is_none()); + // Compared as the response it decodes back into, re-encoded: `serde_json` + // parses floats to the nearest f64 only approximately, so comparing the + // raw `Value`s would report a parser rounding as a lost field. + let data = decoded.data.expect("a success envelope carries data"); + let carried: HealthcheckResponse = serde_json::from_value(data).expect("data decodes"); + prop_assert_eq!( + serde_json::to_string(&carried).expect("re-encode"), + serde_json::to_string(&response).expect("encode"), + "the envelope must carry the caller's data unchanged" + ); + } + + /// Smoke test: mutating a valid encoding must yield either a clean decode + /// or a clean error — never a panic. This can only fail on a panic. + #[test] + fn mutated_valid_encodings_never_panic( + response in healthcheck_strategy(), + hint in recipe_hint_strategy(), + position in any::(), + kind in any::(), + ) { + for json in [ + serde_json::to_string(&response).expect("encode"), + serde_json::to_string(&hint).expect("encode"), + serde_json::to_string(&EngineResponseEnvelope::success(&response)).expect("encode"), + ] { + decode_all_without_panic(&mutate(&json, position, kind)); + } + } + + /// Fully arbitrary bytes must never panic a decoder. + #[test] + fn arbitrary_bytes_never_panic(bytes in prop::collection::vec(any::(), 0..256)) { + decode_all_without_panic(&bytes); + } + + #[test] + fn arbitrary_text_never_panics(text in "[ -~\\n\\t]{0,256}") { + decode_all_without_panic(text.as_bytes()); + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(1024), + ..ProptestConfig::default() + })] + + /// A plugin binary name is attacker-influenced (anyone who can drop a file + /// into a plugin directory). The derived engine id is later used to build + /// paths and command lines, so it must never carry a separator or a + /// `.`/`..` traversal component, and the helper must never panic on a + /// multi-byte name. + #[test] + fn engine_id_is_never_a_traversal_component( + name in prop_oneof![ + "[ -~]{0,24}", + "rocm-engine-[ -~]{0,16}", + "rocm-engine-[A-Za-z0-9._/\\\\-]{0,12}(\\.exe|\\.EXE|)", + "\\PC{0,16}", + // The traversal names themselves, which no random strategy above + // reaches: an id of exactly `.` or `..`, with and without `.exe`. + prop::sample::select(vec![ + "rocm-engine-.", + "rocm-engine-..", + "rocm-engine-..exe", + "rocm-engine-...exe", + "rocm-engine-.EXE", + ]) + .prop_map(str::to_owned), + ], + ) { + if let Some(id) = engine_id_from_plugin_binary_name(&name) { + // The character set below has no separator, so it also rules out + // `/` and `\`. That well-formed names do yield their id, which a + // helper that never returned one would fail, is pinned by + // `plugin_binary_names_yield_their_engine_id`. + prop_assert!(!id.is_empty()); + prop_assert!(id != "." && id != "..", "engine id must not traverse: {id:?}"); + prop_assert!( + id.chars().all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '.' | '_' | '-')), + "engine id must stay in the documented character set: {id:?}" + ); + } + } + + /// `--gpu` is direct user input. Parsing must never panic on arbitrary + /// text, and a successful parse must round-trip through the CLI spelling + /// it reports. Random text almost never parses, so this is mostly a panic + /// check; which values must parse, and to what, is pinned by + /// `gpu_cli_values_parse_as_documented`. + #[test] + fn gpu_selection_parse_never_panics_and_roundtrips(value in "[ -~]{0,16}") { + match GpuSelection::parse_cli_value(&value) { + Ok(GpuSelection::Auto) => { + prop_assert!(launch_gpu_indices(Some(&GpuSelection::Auto)).is_empty()); + } + Ok(GpuSelection::Index(index)) => { + prop_assert_eq!( + GpuSelection::parse_cli_value(&index.to_string()), + Ok(GpuSelection::Index(index)) + ); + prop_assert_eq!(launch_gpu_indices(Some(&GpuSelection::Index(index))), vec![index]); + prop_assert_eq!( + gpu_indices_to_csv(&[index]), + Some(index.to_string()) + ); + } + Err(_) => {} + } + } + + /// A GPU ordinal must never reach `HIP_VISIBLE_DEVICES` as anything but a + /// plain comma-separated list of decimal integers — the value is + /// interpolated into a child process environment. + #[test] + fn gpu_csv_is_always_plain_decimal(indices in prop::collection::vec(any::(), 0..8)) { + match gpu_indices_to_csv(&indices) { + None => prop_assert!(indices.is_empty()), + Some(csv) => { + prop_assert!(!indices.is_empty()); + let parsed: Result, _> = + csv.split(',').map(str::parse::).collect(); + prop_assert_eq!(parsed.as_deref(), Ok(indices.as_slice())); + } + } + } + + /// The loopback allowlist decides whether an endpoint gets an API key. + /// Anything that is not one of the three exact accepted spellings must be + /// classified public (key required) — misclassifying a public bind as + /// loopback opens an unauthenticated listener. + /// + /// A random host string essentially never lands near those spellings, so + /// most cases are drawn from the near-miss region: each loopback spelling + /// padded, re-cased, bracketed or extended, plus other addresses that + /// reach this host but are not on the list. + #[test] + fn only_exact_loopback_spellings_skip_the_api_key( + host in prop_oneof![ + 3 => ( + prop::sample::select(LOOPBACK_SPELLINGS.to_vec()), + 0u8..8, + ) + .prop_map(|(spelling, kind)| near_loopback(spelling, kind)), + 1 => prop::sample::select(OTHER_LOCAL_ADDRESSES.to_vec()).prop_map(str::to_owned), + 1 => "[ -~]{0,24}", + ], + ) { + let loopback = !is_public_bind_host(&host); + let allowlisted = LOOPBACK_SPELLINGS.contains(&host.as_str()); + prop_assert!( + loopback == allowlisted, + "host {host:?} was classified {} but only the exact spellings {LOOPBACK_SPELLINGS:?} \ + may skip the key", + if loopback { "loopback" } else { "public" } + ); + } +} + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(256), + ..ProptestConfig::default() + })] + + /// A `LogsRequest` carries a `tail_lines` count straight from the caller into + /// the engines' tail logic; it must round-trip exactly, including the + /// extremes an engine could legitimately be asked for. + #[test] + fn logs_request_roundtrips( + service_id in "[ -~]{0,24}", + tail_lines in proptest::option::of(prop_oneof![ + Just(0usize), + Just(usize::MAX), + 0usize..10_000, + ]), + ) { + let request = LogsRequest { service_id: service_id.clone(), tail_lines }; + let encoded = serde_json::to_string(&request).expect("encode"); + let decoded: LogsRequest = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(decoded.service_id, service_id); + prop_assert_eq!(decoded.tail_lines, tail_lines); + } + + #[test] + fn logs_response_roundtrips( + log_path in "[ -~]{0,24}", + lines in prop::collection::vec("\\PC{0,40}", 0..8), + ) { + let response = LogsResponse { + log_path: log_path.clone(), + recent_lines: lines.clone(), + }; + let encoded = serde_json::to_string(&response).expect("encode"); + let decoded: LogsResponse = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(decoded.log_path, log_path); + prop_assert_eq!(decoded.recent_lines, lines); + } + + /// `host` and `port` from a launch request are what the CLI later classifies + /// for API-key enforcement, so they must survive the wire unchanged. + #[test] + fn launch_request_roundtrips_host_and_port( + host in "[ -~]{0,24}", + port in any::(), + selection in proptest::option::of(prop_oneof![ + Just(GpuSelection::Auto), + any::().prop_map(GpuSelection::Index), + ]), + device_policy in proptest::option::of(prop_oneof![ + Just(DevicePolicy::GpuRequired), + Just(DevicePolicy::GpuPreferred), + Just(DevicePolicy::CpuOnly), + ]), + ) { + let request = LaunchRequest { + service_id: "svc".to_owned(), + env_id: None, + runtime_id: None, + model_ref: "model".to_owned(), + host: host.clone(), + port, + device_policy: device_policy.clone(), + endpoint_mode: None, + engine_recipe: None, + gpu_selection: selection.clone(), + }; + let encoded = serde_json::to_string(&request).expect("encode"); + let decoded: LaunchRequest = serde_json::from_str(&encoded).expect("decode"); + prop_assert_eq!(&decoded.host, &host); + prop_assert_eq!(decoded.port, port); + prop_assert_eq!(decoded.gpu_selection, selection); + prop_assert_eq!(decoded.device_policy, device_policy); + } + + /// A request envelope decodes exactly when its method is one of the + /// documented wire names — an unknown method is rejected rather than + /// falling back to some default — and a decoded method re-encodes to the + /// spelling it arrived as. Half the cases use a real method name, the rest + /// near misses of one or arbitrary words. + #[test] + fn request_envelope_accepts_exactly_the_known_methods( + method in prop_oneof![ + 2 => prop::sample::select(METHOD_WIRE_NAMES.to_vec()).prop_map(str::to_owned), + 1 => (prop::sample::select(METHOD_WIRE_NAMES.to_vec()), 0u8..4).prop_map( + |(name, kind)| match kind { + 0 => name.to_ascii_uppercase(), + 1 => format!("{name}s"), + 2 => name.replace('_', ""), + _ => format!(" {name}"), + } + ), + 1 => "[a-z_]{0,16}", + ], + payload in "[ -~]{0,24}", + ) { + let json = serde_json::json!({ "method": method, "payload": payload }).to_string(); + let parsed = serde_json::from_str::(&json); + let known = METHOD_WIRE_NAMES.contains(&method.as_str()); + prop_assert_eq!( + parsed.is_ok(), + known, + "method {:?} decoded: {}", + method, + parsed.is_ok() + ); + if let Ok(envelope) = parsed { + let reencoded = serde_json::to_value(&envelope).expect("encode"); + prop_assert_eq!( + reencoded.get("method").and_then(serde_json::Value::as_str), + Some(method.as_str()), + "a decoded method must re-encode to the same wire spelling" + ); + } + } +} + +/// The documented wire name of every engine method, written out rather than +/// derived from the enum. +const METHOD_WIRE_NAMES: [&str; 9] = [ + "detect", + "install", + "capabilities", + "resolve_model", + "launch", + "healthcheck", + "endpoint", + "stop", + "logs", +]; + +/// The three spellings the CLI accepts as loopback, written out here rather than +/// shared with the code under test. +const LOOPBACK_SPELLINGS: [&str; 3] = ["127.0.0.1", "localhost", "::1"]; + +/// Addresses that reach this host, or every interface, without being one of the +/// accepted spellings: all must be treated as public. +const OTHER_LOCAL_ADDRESSES: [&str; 9] = [ + "0.0.0.0", + "::", + "[::1]", + "::ffff:127.0.0.1", + "127.0.0.2", + "127.000.0.1", + "0:0:0:0:0:0:0:1", + "localhost.localdomain", + "", +]; + +/// One near miss of a loopback spelling; kind 0 is the spelling itself. +fn near_loopback(spelling: &str, kind: u8) -> String { + match kind { + 0 => spelling.to_owned(), + 1 => format!(" {spelling}"), + 2 => format!("{spelling}\n"), + 3 => spelling.to_ascii_uppercase(), + 4 => { + let mut chars = spelling.chars(); + chars + .next() + .map(|first| first.to_ascii_uppercase().to_string() + chars.as_str()) + .unwrap_or_default() + } + 5 => format!("[{spelling}]"), + 6 => format!("{spelling}:8080"), + _ => format!("{spelling}."), + } +} + +/// Present four times in five, so that a meaningful share of documents carry +/// every required field and take the decode-succeeds path. +fn mostly_present( + value: impl Strategy + 'static, +) -> impl Strategy> { + proptest::option::weighted(0.8, value) +} + +/// A map strategy over the fields of a valid response, so that fields can be +/// dropped individually — version skew as an engine would produce it, not as +/// random bytes would. +fn field_skew_strategy() -> impl Strategy { + ( + mostly_present("[ -~]{0,8}"), + mostly_present(any::()), + mostly_present("[ -~]{0,8}"), + mostly_present(any::()), + mostly_present(any::()), + "[ -~]{0,8}", + ) + .prop_map( + |(status, model_loaded, device, uptime, queue_depth, extra)| { + let mut object = serde_json::Map::new(); + if let Some(status) = status { + object.insert("status".to_owned(), serde_json::Value::String(status)); + } + if let Some(model_loaded) = model_loaded { + object.insert( + "model_loaded".to_owned(), + serde_json::Value::Bool(model_loaded), + ); + } + if let Some(device) = device { + object.insert("device".to_owned(), serde_json::Value::String(device)); + } + if let Some(uptime) = uptime { + object.insert("uptime_sec".to_owned(), serde_json::Value::from(uptime)); + } + if let Some(queue_depth) = queue_depth { + object.insert( + "queue_depth".to_owned(), + serde_json::Value::from(queue_depth), + ); + } + object.insert( + "unknown_future_field".to_owned(), + serde_json::Value::String(extra), + ); + serde_json::Value::Object(object) + }, + ) +} + +/// The fields `HealthcheckResponse` cannot decode without. +const HEALTHCHECK_REQUIRED_FIELDS: [&str; 5] = [ + "status", + "model_loaded", + "device", + "uptime_sec", + "queue_depth", +]; + +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(256), + ..ProptestConfig::default() + })] + + /// Version skew: a newer engine sends extra fields and may omit ones this + /// build requires. The document decodes exactly when every required field + /// is present — the unknown field it always carries must be ignored — and + /// a decoded response carries the values that were sent. + #[test] + fn version_skew_decodes_exactly_when_required_fields_are_present(value in field_skew_strategy()) { + let bytes = serde_json::to_vec(&value).expect("encode"); + let all_required_present = HEALTHCHECK_REQUIRED_FIELDS + .iter() + .all(|field| value.get(field).is_some()); + let decoded = serde_json::from_slice::(&bytes); + prop_assert_eq!( + decoded.is_ok(), + all_required_present, + "decode result {:?} for {}", + decoded.as_ref().map(|_| ()).map_err(ToString::to_string), + value + ); + if let Ok(response) = decoded { + prop_assert_eq!(Some(response.status.as_str()), value["status"].as_str()); + prop_assert_eq!(Some(response.model_loaded), value["model_loaded"].as_bool()); + prop_assert_eq!(Some(response.device.as_str()), value["device"].as_str()); + prop_assert_eq!(Some(response.uptime_sec), value["uptime_sec"].as_u64()); + prop_assert_eq!( + Some(u64::from(response.queue_depth)), + value["queue_depth"].as_u64() + ); + prop_assert_eq!(response.last_error, None); + } + decode_all_without_panic(&bytes); + } +} + +/// Golden documents for the wire spellings the round trips cannot see: device +/// policy and GPU selection values, and a literal healthcheck response as an +/// engine binary built from another revision would send it. +#[test] +fn protocol_values_use_their_documented_wire_spellings() { + for (policy, wire) in [ + (DevicePolicy::GpuRequired, "gpu_required"), + (DevicePolicy::GpuPreferred, "gpu_preferred"), + (DevicePolicy::CpuOnly, "cpu_only"), + ] { + assert_eq!( + serde_json::to_value(&policy).expect("encode"), + serde_json::json!(wire) + ); + } + assert_eq!( + serde_json::to_value(GpuSelection::Auto).expect("encode"), + serde_json::json!("auto") + ); + assert_eq!( + serde_json::to_value(GpuSelection::Index(1)).expect("encode"), + serde_json::json!({ "index": 1 }) + ); + + let response: HealthcheckResponse = serde_json::from_value(serde_json::json!({ + "status": "ready", + "model_loaded": true, + "device": "gpu", + "uptime_sec": 12, + "queue_depth": 3, + "last_error": "boom", + "tokens_per_sec": 1.5, + })) + .expect("a healthcheck document decodes"); + assert_eq!(response.status, "ready"); + assert!(response.model_loaded); + assert_eq!(response.device, "gpu"); + assert_eq!(response.uptime_sec, 12); + assert_eq!(response.queue_depth, 3); + assert_eq!(response.last_error.as_deref(), Some("boom")); + assert_eq!(response.tokens_per_sec, Some(1.5)); +} + +/// Non-finite token rates have no JSON representation: they encode as `null` +/// and read back as unknown, rather than failing the whole healthcheck. Pinned +/// deterministically, since the round-trip property only samples them. +#[test] +fn non_finite_token_rates_read_back_as_unknown() { + for rate in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + let response: HealthcheckResponse = serde_json::from_value(serde_json::json!({ + "status": "ready", + "model_loaded": true, + "device": "gpu", + "uptime_sec": 1, + "queue_depth": 0, + })) + .expect("decode"); + let response = HealthcheckResponse { + tokens_per_sec: Some(rate), + ..response + }; + let encoded = serde_json::to_string(&response).expect("a non-finite rate still encodes"); + let decoded: HealthcheckResponse = serde_json::from_str(&encoded).expect("decode"); + assert_eq!( + decoded.tokens_per_sec, None, + "{rate} read back as {:?}", + decoded.tokens_per_sec + ); + } +} + +/// `--gpu` values as a user types them, with the selection each must produce. +#[test] +fn gpu_cli_values_parse_as_documented() { + for (value, expected) in [ + ("auto", Ok(GpuSelection::Auto)), + ("AUTO", Ok(GpuSelection::Auto)), + ("", Ok(GpuSelection::Auto)), + (" ", Ok(GpuSelection::Auto)), + ("0", Ok(GpuSelection::Index(0))), + (" 1 ", Ok(GpuSelection::Index(1))), + ("4294967295", Ok(GpuSelection::Index(u32::MAX))), + ] { + assert_eq!(GpuSelection::parse_cli_value(value), expected, "{value:?}"); + } + let multi = GpuSelection::parse_cli_value("1,2").expect_err("multiple GPUs are rejected"); + assert!( + multi.contains("multiple GPUs is not supported"), + "a list of GPUs must say why it is rejected, got {multi:?}" + ); + for value in ["-1", "4294967296", "gpu0", "1.0", "auto1"] { + assert!( + GpuSelection::parse_cli_value(value).is_err(), + "{value:?} must be rejected" + ); + } +} + +/// Literal request documents as the CLI sends them, and a detect response +/// without the optional runtime fields: each must decode to the values it +/// spells. This catches a renamed or removed field; it cannot catch a lost +/// `#[serde(default)]` on an `Option`, which serde treats as optional anyway. +#[test] +fn request_and_detect_documents_have_their_documented_wire_shape() { + let launch: LaunchRequest = serde_json::from_value(serde_json::json!({ + "service_id": "svc", + "env_id": null, + "runtime_id": null, + "model_ref": "org/model", + "host": "127.0.0.1", + "port": 8080, + "device_policy": "gpu_required", + "endpoint_mode": null, + "gpu_selection": { "index": 1 }, + })) + .expect("a launch request decodes"); + assert_eq!(launch.service_id, "svc"); + assert_eq!(launch.model_ref, "org/model"); + assert_eq!(launch.host, "127.0.0.1"); + assert_eq!(launch.port, 8080); + assert_eq!(launch.device_policy, Some(DevicePolicy::GpuRequired)); + assert_eq!(launch.gpu_selection, Some(GpuSelection::Index(1))); + assert!(launch.engine_recipe.is_none()); + + let logs: LogsRequest = + serde_json::from_value(serde_json::json!({ "service_id": "svc", "tail_lines": 50 })) + .expect("a logs request decodes"); + assert_eq!(logs.service_id, "svc"); + assert_eq!(logs.tail_lines, Some(50)); + + let detect: DetectResponse = serde_json::from_value(serde_json::json!({ + "installed": true, + "env_id": "env", + "python_version": null, + "torch_version": null, + "transformers_version": null, + "available_devices": [{ "kind": "gpu", "available": true }], + "capabilities": { + "cpu": false, + "rocm_gpu": true, + "openai_compatible": true, + "tool_calling": false, + "quantized_models": "none", + "reasoning_parser": false, + }, + "notes": ["n"], + })) + .expect("a detect response without the optional runtime fields decodes"); + assert!(detect.installed); + assert_eq!(detect.env_id.as_deref(), Some("env")); + assert!(detect.runtime_kind.is_none() && detect.managed_env.is_none()); + assert_eq!(detect.available_devices.len(), 1); + assert!(detect.available_devices[0].available); + assert!(detect.capabilities.rocm_gpu && !detect.capabilities.cpu); + assert_eq!(detect.notes, vec!["n".to_owned()]); +} + +/// The error envelope an engine sends, as a literal document: a swapped code +/// and message, or a failure that reads as success, must fail here. +#[test] +fn failure_envelope_has_its_documented_wire_shape() { + assert_eq!( + serde_json::to_value(EngineResponseEnvelope::failure("c", "m")).expect("encode"), + serde_json::json!({ "ok": false, "error": { "code": "c", "message": "m" } }) + ); +} + +/// The plugin binary naming rule, positively: the `rocm-engine-` prefix with +/// an optional `.exe` in either case yields the id, and anything else none. +#[test] +fn plugin_binary_names_yield_their_engine_id() { + for name in [ + "rocm-engine-foo", + "rocm-engine-foo.exe", + "rocm-engine-foo.EXE", + ] { + assert_eq!( + engine_id_from_plugin_binary_name(name).as_deref(), + Some("foo"), + "{name}" + ); + } + for name in ["rocm-engin-foo", "engine-foo", "foo", "rocm-engine-"] { + assert_eq!(engine_id_from_plugin_binary_name(name), None, "{name}"); + } +} + +/// The documented wire name of each method. The `match` is exhaustive, so a +/// new method fails to compile until it is listed here; the variant list in +/// the test below and [`METHOD_WIRE_NAMES`] are maintained by hand and are not +/// compile-enforced. +const fn method_wire_name(method: &EngineMethod) -> &'static str { + match method { + EngineMethod::Detect => "detect", + EngineMethod::Install => "install", + EngineMethod::Capabilities => "capabilities", + EngineMethod::ResolveModel => "resolve_model", + EngineMethod::Launch => "launch", + EngineMethod::Healthcheck => "healthcheck", + EngineMethod::Endpoint => "endpoint", + EngineMethod::Stop => "stop", + EngineMethod::Logs => "logs", + } +} + +/// Every method's wire spelling, written out: the plugin protocol is a +/// contract with separately built engine binaries, so a renamed method must +/// fail here rather than at runtime. (The round-trip properties cannot see a +/// rename, since encoding and decoding change together; the golden documents +/// above pin a few more spellings, not the whole wire format.) +#[test] +fn engine_methods_use_their_documented_wire_names() { + for wire in METHOD_WIRE_NAMES { + let envelope: EngineRequestEnvelope = + serde_json::from_value(serde_json::json!({ "method": wire })) + .unwrap_or_else(|error| panic!("{wire:?} must decode: {error}")); + assert_eq!(method_wire_name(&envelope.method), wire); + } + for method in [ + EngineMethod::Detect, + EngineMethod::Install, + EngineMethod::Capabilities, + EngineMethod::ResolveModel, + EngineMethod::Launch, + EngineMethod::Healthcheck, + EngineMethod::Endpoint, + EngineMethod::Stop, + EngineMethod::Logs, + ] { + let wire = method_wire_name(&method); + assert!( + METHOD_WIRE_NAMES.contains(&wire), + "{wire:?} is missing from METHOD_WIRE_NAMES" + ); + assert_eq!( + serde_json::to_value(&method).expect("encode"), + serde_json::Value::from(wire) + ); + let envelope: EngineRequestEnvelope = + serde_json::from_value(serde_json::json!({ "method": wire })) + .unwrap_or_else(|error| panic!("{wire:?} must decode: {error}")); + assert_eq!(envelope.method, method); + } +} + +// `BTreeMap` keys in a recipe hint come off the wire; duplicates in the JSON +// must resolve deterministically rather than depending on iteration order. +proptest! { + #![proptest_config(ProptestConfig { + cases: proptest_cases(256), + ..ProptestConfig::default() + })] + + #[test] + fn duplicate_parser_setting_keys_resolve_last_wins( + key in "[a-z]{1,6}", + first in "[ -~]{0,8}", + second in "[ -~]{0,8}", + ) { + let json = format!( + r#"{{"contract_version":"0.1.0","engine":"vllm","parser_settings":{{{key}:{first},{key}:{second}}}}}"#, + key = serde_json::to_string(&key).expect("key"), + first = serde_json::to_string(&first).expect("first"), + second = serde_json::to_string(&second).expect("second"), + ); + let decoded: EngineRecipeHint = serde_json::from_str(&json).expect("decode"); + let expected: BTreeMap = BTreeMap::from([(key, second)]); + prop_assert_eq!(decoded.parser_settings, expected); + } +} diff --git a/docs/testing.md b/docs/testing.md index ffa0790ae4..c93eca4b76 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -315,11 +315,14 @@ PROPTEST_CASES=10000 cargo test -p rocm --bin rocm proptests ``` An explicit case count overrides `PROPTEST_CASES`, so a module that sets one -reads the variable back itself: through the `proptest_cases` helper in each -crate's test support, [rocm's](../apps/rocm/src/test_support.rs) and -[rocm-core's](../crates/rocm-core/src/test_support.rs), and through -`run_property` in [archive_props.rs](../crates/rocm-core/src/archive_props.rs), -which keeps its own copy because three crates include it by `#[path]`. All of +reads the variable back itself: through the `proptest_cases` helper in +[tests/common/proptest_cases.rs](../crates/rocm-core/tests/common/proptest_cases.rs), +which [rocm's](../apps/rocm/src/test_support.rs) and +[rocm-core's](../crates/rocm-core/src/test_support.rs) test support, the +`rocm-core` and `rocm-engine-protocol` integration tests and the Lemonade +engine's unit tests all include by `#[path]`; and through `run_property` in +[archive_props.rs](../crates/rocm-core/src/archive_props.rs), which keeps its +own copy because three crates include it by `#[path]`. All of them only ever raise the module's count: a `PROPTEST_CASES` below it is ignored, because some of these properties catch their target defect in a small share of cases and would pass with it present on a short run. The uninstall properties read their own @@ -2406,6 +2409,36 @@ because its runtime key and package specs carry the index's spelling. `generator_reach_report` prints how often each hard case is generated (run it with `-- --nocapture`), and fails if the generator stops reaching them. +Artifact integrity and engine protocol property tests: + +```bash +cargo test -p rocm-core --test digest_gate_properties --test integrity_properties --test model_selector_properties +cargo test -p rocm-engine-protocol --test protocol_properties +cargo test -p rocm-engine-lemonade --lib archive_digest +cargo test -p xtask verify_pinned_keys +``` + +These check the code that decides whether a download, a signature or the +pinned release key is authentic, and the code that decodes engine output. The +generators aim at near misses — a digest one nibble off or sharing a correct +prefix, a signature with one byte changed or leading zeros, a loopback spelling +padded or re-cased, a model id with a variant suffix — because a uniformly +random input is rejected by any implementation, correct or not. The pinned-key +tests run the real check over a scratch repository root and walk every +position of the committed key. `model_selector_properties` also pins, rather +than asserts away, two shapes where readiness accepts an engine's reported name +for a different model: a bare family word such as `qwen` reads as ready for +other models of that family, in the catalog and beyond, and a catalog alias +that spells another catalog model's basename (`qwen3-32b`, an alias of +`Qwen/Qwen3-32B-FP8` and the basename of `Qwen/Qwen3-32B`) reads as ready for +that other model. The second is checked against a list of the catalog's +current collisions, so a catalog change that adds one fails the test. +The pinned-key tests also pin that a character outside the base64 alphabet +(a space, `!` or `.`) inserted into an installer's copy of the key is refused +as a malformed key block, in either installer, rather than ignored by the +comparison: an installer that embeds such a key cannot load it. +`non_base64_noise_in_an_embedded_key_is_refused_as_malformed` holds that. + Model recipe engine metadata checks: ```bash diff --git a/engines/lemonade/src/archive_cache.rs b/engines/lemonade/src/archive_cache.rs index ad2b7177ce..99012c2ef2 100644 --- a/engines/lemonade/src/archive_cache.rs +++ b/engines/lemonade/src/archive_cache.rs @@ -283,7 +283,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::test_support::scratch_dir; + use crate::test_support::{proptest_cases, scratch_dir}; use std::io::Write; use std::process::{Command as ProcessCommand, Stdio}; @@ -300,6 +300,144 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// Whether `error` is the digest check's own rejection, so a failure to + /// read the archive cannot pass as one. + fn is_digest_mismatch(error: &anyhow::Error) -> bool { + error.to_string().contains("SHA-256 mismatch") + } + + /// A runner with a fixed seed, so a failure reproduces as-is, running at + /// least `cases` cases. `TestRunner::deterministic` alone takes its case + /// count from `PROPTEST_CASES` unchecked, so `PROPTEST_CASES=0` would pass + /// these properties without running one. + fn deterministic_runner(cases: u32) -> proptest::test_runner::TestRunner { + let config = proptest::test_runner::Config { + cases: proptest_cases(cases), + ..proptest::test_runner::Config::default() + }; + let rng = proptest::test_runner::TestRng::deterministic_rng(config.rng_algorithm); + proptest::test_runner::TestRunner::new_with_rng(config, rng) + } + + /// Property: the archive digest check accepts the pinned digest and nothing + /// else. + /// + /// The generator targets near misses rather than random strings, because a + /// random 64-character candidate is rejected by any implementation — + /// correct, prefix-matching, or truncating alike. What separates them is a + /// candidate that shares most of the real digest: one nibble changed, the + /// correct prefix with a wrong tail, a truncated prefix, or the real digest + /// with whitespace or an extra character attached. + /// + /// Note the comparison is byte-exact, so an upper-case spelling of the + /// *correct* digest is also rejected. That is fail-closed (an upper-case + /// pin would break installs, not admit a bad archive) but it differs from + /// `rocm_core`'s download gate, which folds ASCII case. + #[test] + fn archive_digest_check_accepts_only_the_exact_pinned_digest() { + use proptest::prelude::*; + + let dir = scratch_dir("archive-digest-properties"); + let archive = dir.join("archive"); + let content = b"pinned embeddable archive"; + fs::write(&archive, content).unwrap(); + let truth = format!("{:x}", Sha256::digest(content)); + + assert!(verify_sha256(&archive, &truth).is_ok()); + + let mut runner = deterministic_runner(256); + let near_miss = { + let truth = truth.clone(); + (0usize..64, 1usize..64, 0u8..7).prop_map(move |(position, keep, kind)| match kind { + 0 => { + let mut out: Vec = truth.chars().collect(); + out[position] = if out[position] == 'a' { 'b' } else { 'a' }; + out.into_iter().collect::() + } + 1 => format!("{}{}", &truth[..keep], "0".repeat(64 - keep)), + 2 => truth[..keep].to_owned(), + 3 => format!("{truth}\n"), + 4 => format!(" {truth} "), + 5 => format!("{truth}0"), + _ => truth.to_ascii_uppercase(), + }) + }; + + runner + .run(&near_miss, |candidate| { + prop_assert_ne!( + &candidate, + &truth, + "the generator must only produce near misses" + ); + prop_assert!( + verify_sha256(&archive, &candidate) + .is_err_and(|error| is_digest_mismatch(&error)), + "the archive digest check accepted {:?} for an archive whose digest is {:?}", + candidate, + truth + ); + Ok(()) + }) + .unwrap(); + + // An empty pin is the one near miss the generator above cannot produce + // (it keeps at least one character); it must not read as "no check". + assert!( + verify_sha256(&archive, "").is_err_and(|error| is_digest_mismatch(&error)), + "an empty pinned digest accepted the archive" + ); + + fs::remove_dir_all(&dir).ok(); + } + + /// Property: flipping a bit of the archive — at sampled positions, and in + /// the last byte — invalidates it against the pinned digest: the + /// tampered-download case. The archive is longer + /// than any single read buffer, and the intact archive must still be + /// accepted: a check that hashed only a prefix would reject the intact + /// archive and miss flips near the end. + #[test] + fn archive_digest_check_rejects_a_single_bit_of_tampering() { + use proptest::prelude::*; + + let dir = scratch_dir("archive-digest-tamper"); + let archive = dir.join("archive"); + let content: Vec = (0..=255_u8).cycle().take(64 * 1024 + 1).collect(); + let truth = format!("{:x}", Sha256::digest(&content)); + + fs::write(&archive, &content).unwrap(); + assert!( + verify_sha256(&archive, &truth).is_ok(), + "the intact multi-buffer archive must be accepted" + ); + // The last byte is checked explicitly: random positions would almost + // never land on it. + let mut tail_flipped = content.clone(); + *tail_flipped.last_mut().unwrap() ^= 1; + fs::write(&archive, &tail_flipped).unwrap(); + assert!( + verify_sha256(&archive, &truth).is_err_and(|error| is_digest_mismatch(&error)), + "a flip in the last byte was accepted" + ); + + let mut runner = deterministic_runner(256); + runner + .run(&(0usize..content.len(), 0u8..8), |(position, bit)| { + let mut tampered = content.clone(); + tampered[position] ^= 1 << bit; + fs::write(&archive, &tampered).unwrap(); + prop_assert!( + verify_sha256(&archive, &truth).is_err_and(|error| is_digest_mismatch(&error)), + "an archive differing by one bit was accepted" + ); + Ok(()) + }) + .unwrap(); + + fs::remove_dir_all(&dir).ok(); + } + #[test] fn repeated_valid_archive_publication_is_idempotent() { let dir = scratch_dir("valid-publication-race"); diff --git a/engines/lemonade/src/test_support.rs b/engines/lemonade/src/test_support.rs index 7b79057c86..7a604084ef 100644 --- a/engines/lemonade/src/test_support.rs +++ b/engines/lemonade/src/test_support.rs @@ -8,6 +8,10 @@ use std::fs; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; +#[path = "../../../crates/rocm-core/tests/common/proptest_cases.rs"] +mod proptest_cases; +pub(crate) use proptest_cases::proptest_cases; + /// A fresh, empty scratch directory under the crate's `target/`, named /// `lemonade-fs-test----`. /// diff --git a/xtask/src/verify_pinned_keys.rs b/xtask/src/verify_pinned_keys.rs index 5f96f52ceb..752cf97a76 100644 --- a/xtask/src/verify_pinned_keys.rs +++ b/xtask/src/verify_pinned_keys.rs @@ -2416,4 +2416,477 @@ mod tests { "an unset variable must resolve to None, not to some other variable" ); } + + /// The canonical current release key exactly as committed, so the checks + /// below run against the real trust root rather than a fixture. + fn canonical_release_key_text() -> String { + std::fs::read_to_string(repo_root().join(PINNED_KEYS[0].canonical)) + .expect("the canonical current release key is committed") + } + + /// The canonical PEM's base64 lines, taken from the raw file text by line + /// position (everything between the header and footer lines) rather than + /// through `pem_body`, so a defect in the normalization under test cannot + /// also shape the expected value. + fn raw_body_lines(pem: &str) -> Vec { + let lines: Vec<&str> = pem.lines().collect(); + assert_eq!( + lines.first(), + Some(&PEM_BEGIN), + "canonical key starts with its header" + ); + let footer = lines + .iter() + .position(|line| *line == PEM_END) + .expect("canonical key has a footer"); + lines[1..footer] + .iter() + .map(|line| (*line).to_owned()) + .collect() + } + + fn pem_from_lines(body: &[String]) -> String { + format!("{PEM_BEGIN}\n{}\n{PEM_END}\n", body.join("\n")) + } + + /// The two installer constants the current release key is pinned in, in the + /// shape the real installers embed them, including the `readonly` line the + /// check requires after the assignments. + fn install_sh(current: &str, extra: &str) -> String { + format!( + "#!/bin/sh\n{extra}PINNED_RELEASE_PUBLIC_KEY_CURRENT=\"{current}\"\n\ + PINNED_RELEASE_PUBLIC_KEY_NEXT=\"\"\n\ + readonly PINNED_RELEASE_PUBLIC_KEY_CURRENT PINNED_RELEASE_PUBLIC_KEY_NEXT\n" + ) + } + + fn install_ps1(current: &str, extra: &str) -> String { + // An empty slot is the one-line `""` form, as in a dormant installer; an + // empty here-string is a malformed key block, not an empty slot. + let slot = if current.is_empty() { + "$PinnedReleasePublicKeyCurrent = \"\"\n".to_owned() + } else { + format!("$PinnedReleasePublicKeyCurrent = @\"\n{current}\"@\n") + }; + format!("{extra}{slot}$PinnedReleasePublicKeyNext = \"\"\n") + } + + /// A repository root holding the real canonical current release key and the + /// given installer sources, for running [`check_pinned_keys`] — the real + /// comparison — end to end. The other canonical keys are absent, so their + /// entries are dormant; the check still reads the metadata key's Rust + /// source to confirm its slot is empty, so the layout holds one. + fn release_layout(canonical: &str, sh: &str, ps1: &str) -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::create_dir_all(dir.path().join("docs/keys")).expect("create docs/keys"); + std::fs::write(dir.path().join(PINNED_KEYS[0].canonical), canonical) + .expect("write canonical key"); + let rust = dir.path().join(rs_path()); + std::fs::create_dir_all(rust.parent().expect("a parent")) + .expect("create the Rust source dir"); + std::fs::write(rust, "const PINNED_METADATA_PUBLIC_KEY_PEM: &str = \"\";\n") + .expect("write the dormant Rust source"); + std::fs::write(dir.path().join("install.sh"), sh).expect("write install.sh"); + std::fs::write(dir.path().join("install.ps1"), ps1).expect("write install.ps1"); + dir + } + + fn diverges(result: &Result>) -> bool { + result + .as_ref() + .is_err_and(|error| error.to_string().contains("have diverged")) + } + + /// Rewrite the installer sources of a [`release_layout`] in place, so a loop + /// over many cases reuses one directory instead of creating one per case. + fn write_sources(root: &Path, sh: &str, ps1: &str) { + std::fs::write(root.join("install.sh"), sh).expect("write install.sh"); + std::fs::write(root.join("install.ps1"), ps1).expect("write install.ps1"); + } + + const BASE64_ALPHABET: &str = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + /// The single-character edits applied at one position of a body line: + /// deletion, duplication, substitution by each neighbour in the base64 + /// alphabet (which swaps `+` and `/`, the two symbols a letters-and-digits + /// normalization would drop; `=` padding has none), for a letter the same + /// letter in the other case, and a swap with the next character when the + /// two differ (which a comparison that ignores character order would miss). + fn single_character_edits(line: &str, column: usize) -> Vec { + let alphabet: Vec = BASE64_ALPHABET.chars().collect(); + let chars: Vec = line.chars().collect(); + let original = chars[column]; + // `=` padding has no alphabet neighbours; deletion and duplication + // still apply to it. (The committed key carries no padding today, so + // this only matters after a rotation to one that does.) + let mut replacements = alphabet + .iter() + .position(|&c| c == original) + .map(|index| { + vec![ + alphabet[(index + 1) % alphabet.len()], + alphabet[(index + alphabet.len() - 1) % alphabet.len()], + ] + }) + .unwrap_or_default(); + if original.is_ascii_alphabetic() { + replacements.push(if original.is_ascii_uppercase() { + original.to_ascii_lowercase() + } else { + original.to_ascii_uppercase() + }); + } + + let mut edits: Vec = replacements + .into_iter() + .map(|replacement| { + let mut edited = chars.clone(); + edited[column] = replacement; + edited.into_iter().collect() + }) + .collect(); + let mut deleted = chars.clone(); + deleted.remove(column); + edits.push(deleted.into_iter().collect()); + let mut duplicated = chars.clone(); + duplicated.insert(column, original); + edits.push(duplicated.into_iter().collect()); + if chars.get(column + 1).is_some_and(|&next| next != original) { + let mut swapped = chars; + swapped.swap(column, column + 1); + edits.push(swapped.into_iter().collect()); + } + edits + } + + /// Every single-character drift of the pinned current release key — at + /// every position of the real published key, in either installer — is + /// reported as divergence by the real check. + /// + /// This is the guarantee the whole check exists for: installers must pin + /// exactly the key CI signs releases with, so a one-character slip during a + /// rotation cannot pass. A random key body is caught by any comparison at + /// all; what separates a correct one from a prefix, truncating or + /// over-normalizing comparison is a body that matches everywhere but one + /// place. So rather than sampling, this walks every position and applies + /// each edit [`single_character_edits`] produces there. Look-alike swaps + /// beyond those (`l`/`I`, `O`/`0`) are not enumerated. + #[test] + fn every_single_character_drift_of_the_pinned_key_is_detected() { + let canonical = canonical_release_key_text(); + let body = raw_body_lines(&canonical); + + // Positive control: the untampered layout passes, so every failure below + // is the drift being detected, not the layout being broken. + let root = release_layout( + &canonical, + &install_sh(&canonical, ""), + &install_ps1(&canonical, ""), + ); + let messages = check_pinned_keys(root.path(), None).expect("the intact layout passes"); + assert!( + messages + .iter() + .any(|m| m.starts_with("release-current: pinned in")), + "the intact layout must actually compare the current key, got {messages:?}" + ); + + let mut expected_checks = 0_usize; + let mut checked = 0_usize; + for (line, text) in body.iter().enumerate() { + for (column, original) in text.chars().enumerate() { + let edits = single_character_edits(text, column); + let swaps = usize::from( + text.chars() + .nth(column + 1) + .is_some_and(|next| next != original), + ); + expected_checks += 2 + * (swaps + + match original { + '=' => 2, + letter if letter.is_ascii_alphabetic() => 5, + _ => 4, + }); + for edit in edits { + let mut tampered_body = body.clone(); + tampered_body[line] = edit; + let tampered = pem_from_lines(&tampered_body); + for (sh, ps1, which) in [ + ( + install_sh(&tampered, ""), + install_ps1(&canonical, ""), + "install.sh", + ), + ( + install_sh(&canonical, ""), + install_ps1(&tampered, ""), + "install.ps1", + ), + ] { + write_sources(root.path(), &sh, &ps1); + let result = check_pinned_keys(root.path(), None); + assert!( + diverges(&result), + "{which}: a key differing at line {line}, column {column} was not \ + reported as divergence: {result:?}" + ); + checked += 1; + } + } + } + } + // The walk covered the whole key, counted straight from the committed + // file rather than through `raw_body_lines`, whatever its size: a + // rotation to a different key type or length changes this count, not + // the test. + let walked: usize = body.iter().map(|line| line.chars().count()).sum(); + let committed: usize = canonical + .lines() + .filter(|line| !line.starts_with("-----")) + .map(|line| line.chars().count()) + .sum(); + assert!(committed > 0, "the committed key has a body"); + assert_eq!( + walked, committed, + "the walk must cover every body character of the committed key" + ); + assert_eq!(checked, expected_checks, "every position gets every edit"); + } + + /// A published canonical key whose pinned constant is empty, or missing + /// from the source altogether, is an error — not a match and not a dormant + /// skip. An empty constant is divergence; a missing one is reported as a + /// missing assignment. Each case leaves exactly one entry wrong, so the + /// error must name that entry. + #[test] + fn an_empty_or_missing_constant_against_a_published_key_is_an_error() { + let canonical = canonical_release_key_text(); + let root = release_layout( + &canonical, + &install_sh(&canonical, ""), + &install_ps1(&canonical, ""), + ); + let next_canonical = root.path().join(PINNED_KEYS[1].canonical); + + let cases = [ + ( + "current empty in install.sh", + install_sh("", ""), + install_ps1(&canonical, ""), + false, + "release-current", + "have diverged", + ), + ( + "current empty in install.ps1", + install_sh(&canonical, ""), + install_ps1("", ""), + false, + "release-current", + "have diverged", + ), + ( + "current constant missing from install.sh", + "#!/bin/sh\nPINNED_RELEASE_PUBLIC_KEY_NEXT=\"\"\n".to_owned(), + install_ps1(&canonical, ""), + false, + "release-current", + "has no assignment of PINNED_RELEASE_PUBLIC_KEY_CURRENT", + ), + // The rotation case: the next key's PEM is published (the same key + // stands in for it) while the installers still carry `NEXT=""`. + ( + "next published but empty in the installers", + install_sh(&canonical, ""), + install_ps1(&canonical, ""), + true, + "release-next", + "have diverged", + ), + ]; + for (label, sh, ps1, publish_next, entry, reported_as) in cases { + if publish_next { + std::fs::write(&next_canonical, &canonical).expect("write canonical next key"); + } else { + let _ = std::fs::remove_file(&next_canonical); + } + write_sources(root.path(), &sh, &ps1); + let result = check_pinned_keys(root.path(), None); + let message = result + .as_ref() + .expect_err(&format!("{label}: expected an error, got {result:?}")) + .to_string(); + assert!( + message.contains(reported_as), + "{label}: expected the error to say `{reported_as}`, got {message}" + ); + assert!( + message.starts_with(&format!("{entry}:")), + "{label}: the error must be reported for {entry}, got {message}" + ); + } + } + + /// An empty canonical file is a key not yet published (dormant), while a + /// pinned source file that does not exist is an error naming the file — + /// neither may read as a match. + #[test] + fn a_blank_canonical_key_is_dormant_and_a_missing_source_is_an_error() { + let canonical = canonical_release_key_text(); + // A dormant slot is empty in every installer; a key pasted into one is + // its own error (`a_dormant_slot_must_be_empty`). + let root = release_layout(" \n", &install_sh("", ""), &install_ps1("", "")); + let messages = + check_pinned_keys(root.path(), None).expect("a blank canonical key is dormant"); + assert!( + messages.iter().any(|m| m + == "release-current: no canonical key yet — slots empty in install.sh, \ + install.ps1 (dormant)"), + "a blank canonical key must be reported dormant, got {messages:?}" + ); + + // Publish the key and pin it in install.sh, so the missing install.ps1 + // is the only fault left. + std::fs::write(root.path().join(PINNED_KEYS[0].canonical), &canonical) + .expect("write canonical key"); + std::fs::write(root.path().join("install.sh"), install_sh(&canonical, "")) + .expect("pin the key in install.sh"); + std::fs::remove_file(root.path().join("install.ps1")).expect("remove install.ps1"); + let error = check_pinned_keys(root.path(), None) + .expect_err("a missing pinned source must be an error"); + assert_eq!( + error.to_string(), + "release-current: source file is missing: install.ps1" + ); + } + + /// A correct copy of the canonical key elsewhere in the same file — in a + /// comment, in another constant, in a stale copy, before or after the + /// constant — never makes a wrong `CURRENT` constant pass. + /// + /// The decoys are placed *before* the constant as well as after it, because + /// a check that reads the first key in the file would only be caught by a + /// decoy that comes first. + #[test] + fn a_correct_key_elsewhere_in_the_file_never_rescues_a_wrong_constant() { + let canonical = canonical_release_key_text(); + let mut body = raw_body_lines(&canonical); + // Flip the case of the first letter on the first body line, a + // one-character drift that stays inside the base64 alphabet. + let flipped: String = { + let mut chars: Vec = body[0].chars().collect(); + let at = chars + .iter() + .position(char::is_ascii_alphabetic) + .expect("a letter to flip"); + chars[at] = if chars[at].is_ascii_uppercase() { + chars[at].to_ascii_lowercase() + } else { + chars[at].to_ascii_uppercase() + }; + chars.into_iter().collect() + }; + body[0] = flipped; + let wrong = pem_from_lines(&body); + assert_fixture_differs(&wrong, &canonical); + + let sh_decoys = [ + format!("# previous key:\n# {}\n", canonical.replace('\n', "\n# ")), + format!("PREVIOUS_RELEASE_PUBLIC_KEY=\"{canonical}\"\n"), + format!("STALE_COPY=\"{canonical}\"\n"), + ]; + let ps1_decoys = [ + format!("# previous key:\n# {}\n", canonical.replace('\n', "\n# ")), + format!("$PreviousReleasePublicKey = @\"\n{canonical}\"@\n"), + format!("$StaleCopy = \"{canonical}\"\n"), + ]; + + for decoy in &sh_decoys { + for sh in [ + install_sh(&wrong, decoy), + format!("{}{decoy}", install_sh(&wrong, "")), + ] { + let root = release_layout(&canonical, &sh, &install_ps1(&canonical, "")); + let result = check_pinned_keys(root.path(), None); + assert!( + diverges(&result), + "install.sh: a wrong CURRENT passed because the canonical key appears \ + elsewhere ({decoy:?}): {result:?}" + ); + } + } + for decoy in &ps1_decoys { + for ps1 in [ + install_ps1(&wrong, decoy), + format!("{}{decoy}", install_ps1(&wrong, "")), + ] { + let root = release_layout(&canonical, &install_sh(&canonical, ""), &ps1); + let result = check_pinned_keys(root.path(), None); + assert!( + diverges(&result), + "install.ps1: a wrong CURRENT passed because the canonical key appears \ + elsewhere ({decoy:?}): {result:?}" + ); + } + } + } + + fn assert_fixture_differs(left: &str, right: &str) { + assert_ne!(left, right, "the fixture must actually differ"); + } + + /// A character outside the base64 alphabet inserted into an installer's + /// copy of the key — whitespace or punctuation — is refused as a malformed + /// key block, in either installer, and the error names the entry and the + /// file. + /// + /// The comparison itself only looks at the base64 alphabet, so this is the + /// well-formedness rule (`pinned_pem_body`) doing the work. It has to: an + /// installer that embeds such a key cannot load it (OpenSSL refuses a `!` or + /// `.`, and `Convert.FromBase64String` throws on anything outside the + /// alphabet), so it would reject every release signed with that key while a + /// comparison alone would pass. + #[test] + fn non_base64_noise_in_an_embedded_key_is_refused_as_malformed() { + let canonical = canonical_release_key_text(); + for noise in [" ", "!", "."] { + // After the first character of the body, which every key has, rather + // than at the start of a line: install.sh refuses a key line that + // starts with whitespace, and that is a separate rule. + let first = canonical + .strip_prefix(&format!("{PEM_BEGIN}\n")) + .and_then(|body| body.chars().next()) + .expect("a body character after the BEGIN line"); + let noisy = canonical.replacen( + &format!("{PEM_BEGIN}\n{first}"), + &format!("{PEM_BEGIN}\n{first}{noise}"), + 1, + ); + assert_fixture_differs(&noisy, &canonical); + for (file, sh, ps1) in [ + ( + "install.sh", + install_sh(&noisy, ""), + install_ps1(&canonical, ""), + ), + ( + "install.ps1", + install_sh(&canonical, ""), + install_ps1(&noisy, ""), + ), + ] { + let root = release_layout(&canonical, &sh, &ps1); + let error = check_pinned_keys(root.path(), None) + .expect_err("a key with stray characters in its body must be refused") + .to_string(); + assert!( + error.starts_with(&format!("release-current: {file} ")) + && error.contains("is not one well-formed PUBLIC KEY PEM block"), + "{noise:?} in {file}: expected a malformed-block error naming {file}, got {error}" + ); + } + } + } }