diff --git a/crates/rocm-core/src/test_env.rs b/crates/rocm-core/src/test_env.rs index e1af4d09b..e12d26cb5 100644 --- a/crates/rocm-core/src/test_env.rs +++ b/crates/rocm-core/src/test_env.rs @@ -64,3 +64,152 @@ impl Drop for RestoredEnvVar { } } } + +#[cfg(test)] +mod tests { + use super::RestoredEnvVar; + use std::ffi::OsString; + use std::marker::PhantomData; + use std::path::Path; + use std::sync::MutexGuard; + + /// Serializes the env mutation below, as the contract guard requires. + static TEST_ENV_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + /// Used by nothing else in the tree, so no other test reads it mid-flight. + const KEY: &str = "ROCM_CORE_TEST_ENV_RESTORE_PROBE"; + + /// Leaves [`KEY`] unset when a test exits, including by a failed assertion. + /// + /// It borrows the lock guard it was armed with, so it drops while that + /// lock is still held: releasing the lock first does not compile. A test + /// about not leaking environment state should not leak the probe when it + /// fails, which is exactly when someone would be looking at it. + /// + /// Holding one mutates the environment from a `Drop` impl the contract + /// guard cannot see, so the call that builds one, `UnsetKeyOnExit::arm(`, + /// is named in its `MUTATIONS` list: a test that arms one without taking a + /// `*_TEST_LOCK` is flagged like a direct call. Build it only through + /// `arm`: constructing it directly (`UnsetKeyOnExit(PhantomData)`) skips + /// both the lock borrow and the text the scan matches. + struct UnsetKeyOnExit<'lock>(PhantomData<&'lock ()>); + + impl<'lock> UnsetKeyOnExit<'lock> { + /// Borrows the caller's lock guard for as long as the returned value + /// lives, so arming one without holding a lock, or releasing the lock + /// before this drops, does not compile. Any `Mutex<()>` guard + /// satisfies the type; that some `*_TEST_LOCK` is taken stays the + /// contract guard's to check (it does not check which one). + const fn arm(_lock: &'lock MutexGuard<'_, ()>) -> Self { + Self(PhantomData) + } + } + + impl Drop for UnsetKeyOnExit<'_> { + #[allow(unsafe_code)] // std::env::remove_var is unsafe in edition 2024 + fn drop(&mut self) { + // SAFETY: dropped inside the caller's `TEST_ENV_TEST_LOCK` scope, + // and nothing else reads this key. + unsafe { std::env::remove_var(KEY) }; + } + } + + /// The restore is this type's whole purpose, and it is what every caller + /// relies on to keep a panicking test from leaking a value into the next + /// one. + /// + /// Both prior states are needed, because they catch different mistakes: + /// + /// * *previously unset* must come back **unset**, not merely empty — which a + /// `var().unwrap_or_default()`-style capture would get wrong — and it also + /// catches capturing the previous value *after* the mutation, which would + /// "restore" the new value and leave the key set; + /// * *previously set* must come back to that exact value, which a `Drop` + /// that does nothing would leave as the planted one. + /// + /// It ends by dropping its [`UnsetKeyOnExit`] by hand, with the probe still + /// set, and asserting the probe is gone: the cleanup is a claim too. + #[test] + #[allow(unsafe_code)] // std::env::set_var/remove_var are unsafe in edition 2024 + fn dropping_the_guard_restores_the_previous_state() { + let lock = TEST_ENV_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let unset_on_exit = UnsetKeyOnExit::arm(&lock); + + // SAFETY: serialized by the lock above, and nothing else reads this key. + unsafe { std::env::remove_var(KEY) }; + drop(RestoredEnvVar::set(KEY, Path::new("planted"))); + assert_eq!( + std::env::var_os(KEY), + None, + "a key that was unset must be unset again, not empty or still planted" + ); + + // SAFETY: as above. + unsafe { std::env::set_var(KEY, "before") }; + let restore = RestoredEnvVar::set(KEY, Path::new("planted")); + assert_eq!( + std::env::var_os(KEY).as_deref(), + Some(std::ffi::OsStr::new("planted")), + "the guard must actually set the value it was given" + ); + drop(restore); + assert_eq!( + std::env::var_os(KEY).as_deref(), + Some(std::ffi::OsStr::new("before")), + "a key that was set must get its previous value back" + ); + + drop(unset_on_exit); + assert_eq!( + std::env::var_os(KEY), + None, + "dropping the cleanup guard must leave the probe unset" + ); + } + + /// The path the module docs promise: a test that panics while holding the + /// guard still gets its previous value back. An explicit `drop` runs the + /// same destructor, but only an unwind exercises it while the thread is + /// panicking: a `Drop` that skips the restore when + /// [`std::thread::panicking`] is true passes + /// `dropping_the_guard_restores_the_previous_state` and fails this test. + /// + /// The panic is raised with [`std::panic::resume_unwind`], which unwinds + /// without calling the panic hook, so a passing run prints no panic message + /// and no process-wide hook has to be swapped while other tests run. + /// + /// The payload carries the value observed just before the panic. Without + /// it, `catch_unwind` would accept any panic — including one raised by + /// `set` itself before it planted anything — and a `set` that planted + /// nothing would leave `"before"` in place for the final assertion to + /// pass on. An assertion inside the closure would not help: `catch_unwind` + /// swallows it and the result is still an `Err`. + #[test] + #[allow(unsafe_code)] // std::env::set_var is unsafe in edition 2024 + fn unwinding_past_the_guard_restores_the_previous_state() { + let lock = TEST_ENV_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _unset_on_exit = UnsetKeyOnExit::arm(&lock); + + // SAFETY: serialized by the lock above, and nothing else reads this key. + unsafe { std::env::set_var(KEY, "before") }; + let unwound = std::panic::catch_unwind(|| { + let _restore = RestoredEnvVar::set(KEY, Path::new("planted")); + std::panic::resume_unwind(Box::new(std::env::var_os(KEY))); + }); + + assert_eq!( + unwound.unwrap_err().downcast_ref::>(), + Some(&Some(OsString::from("planted"))), + "the closure must have unwound, with the guard's value in place" + ); + assert_eq!( + std::env::var_os(KEY).as_deref(), + Some(std::ffi::OsStr::new("before")), + "unwinding past the guard must put the previous value back" + ); + } +} diff --git a/docs/testing.md b/docs/testing.md index a21272ebc..9ca9d99bc 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -131,7 +131,9 @@ The guard accepts `ScopedTestEnv` or **any** static named `*_TEST_LOCK` taken inside the test body. The suffix is the rule, so a new lock is recognised the day it is declared rather than when someone remembers to add it to a list. The exemption is per test function, not per file: one test taking -a lock does not cover its neighbours. +a lock does not cover its neighbours. "Taken" means acquired: a `*_TEST_LOCK` +counts only with a `.lock(` in the same statement, so `let _x = &SOME_TEST_LOCK;` +names a lock without holding it and does not satisfy the guard. Two things the shape above gets right, both of which the guard checks only partly: @@ -150,14 +152,19 @@ it reads what the panicking one left behind. `rocm-core`'s `RestoredEnvVar` unwinding runs it; prefer it where one exists. It restores but does not serialize, so the lock is still yours to take. -Keep the mutation and the lock in the same test body. Three known blind spots, -all of which the scan reports nothing for: +Keep the mutation and the lock in the same test body. The scan reports nothing +for any of these known blind spots: - A `#[test]` that delegates its mutation to a helper — the helper's body is a different scope, so the scan cannot see the two together. It matches call text, so a wrapper hides the mutation until the wrapper itself is named in the scanner's `MUTATIONS` list, as `RestoredEnvVar::set(` is. Add a new restoring helper to that list, or it is outside the guard. +- A guard held by the test whose `Drop` mutates the environment — the mutation + runs in the `Drop` impl, which the scan never enters. List the call that + builds the guard in `MUTATIONS`, as `UnsetKeyOnExit::arm(` and + `EnvVarGuard::set(` are, and build it only through that call: constructing + the type directly leaves no listed text to match, and nothing enforces this. - A harness attribute that does not end in `test`, such as `#[test_case(..)]` or `#[rstest]`. Neither is used in this tree. - `#[cfg_attr(unix, test)]`, for the same reason: the attribute's path reads as diff --git a/xtask/src/env_mutation_contract.rs b/xtask/src/env_mutation_contract.rs index 41f2353a6..5d373c1a9 100644 --- a/xtask/src/env_mutation_contract.rs +++ b/xtask/src/env_mutation_contract.rs @@ -107,13 +107,33 @@ mod tests { /// an offense. Both directions have fixtures. /// `RestoredEnvVar::set(` is here because the guard matches TEXT, and a /// mutation spelled as a method on a restoring wrapper contains neither - /// direct call. The two tests that use it are the sanctioned exception this - /// change documents, and before this entry existed they sat outside the - /// guard entirely — deleting their lock acquisition left the scan green, - /// which is the opposite of what their comments told the next contributor. + /// direct call. The tests that use it are the sanctioned exception, and + /// before this entry existed they sat outside the guard entirely — + /// deleting their lock acquisition left the scan green, which is the + /// opposite of what their comments told the next contributor. /// Listing the helper keeps the escape hatch narrow: it is still the lock /// that is required, the wrapper just stops hiding the requirement. - const MUTATIONS: [&str; 3] = ["set_var(", "remove_var(", "RestoredEnvVar::set("]; + /// + /// `UnsetKeyOnExit::arm(` (`crates/rocm-core/src/test_env.rs`) is here + /// for the same reason, from the other side: `UnsetKeyOnExit` is a guard + /// whose `Drop` removes a variable, so the mutation happens in an impl the + /// scan never enters. What the test body does contain is the call that + /// builds the guard, and that call is what is listed. A guard constructed + /// directly from its type, rather than through that call, would leave + /// nothing for this list's text to match; build such a guard only through + /// the listed call. + /// + /// `EnvVarGuard::set(` (`crates/rocm-core/src/examine/framework.rs`) is the + /// same shape: its constructor sets the variable and its `Drop` restores + /// it, both in impls the scan never enters, so the test that builds one + /// shows neither call. + const MUTATIONS: [&str; 5] = [ + "set_var(", + "remove_var(", + "RestoredEnvVar::set(", + "UnsetKeyOnExit::arm(", + "EnvVarGuard::set(", + ]; /// Named helpers that serialize env mutation for their whole scope. /// @@ -489,6 +509,41 @@ mod tests { ); } + /// Every entry in [`MUTATIONS`] still names a call made somewhere in the + /// tree. + /// + /// The entries are text, so renaming a wrapper or guard leaves its entry + /// matching nothing: the scan stays green while the renamed call is + /// outside it, which is the silent failure this module is written + /// against. Matched on code with literals and comments stripped, so a + /// mention in a doc comment does not keep a dead entry alive, and this + /// file is skipped because its fixtures exist to spell the calls. + #[test] + fn every_listed_mutation_is_still_called_in_the_tree() { + let this_file = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/env_mutation_contract.rs"); + let code: Vec = workspace_rust_sources() + .iter() + .filter(|path| **path != this_file) + .map(|path| { + let text = std::fs::read_to_string(path) + .unwrap_or_else(|e| panic!("reading {}: {e}", path.display())); + strip_literals_and_comments(&text) + }) + .collect(); + + let stale: Vec<&str> = MUTATIONS + .iter() + .copied() + .filter(|call| !code.iter().any(|text| text.contains(call))) + .collect(); + assert!( + stale.is_empty(), + "MUTATIONS lists calls nothing in the tree makes any more, so the \ + scan no longer covers whatever replaced them -- rename the entry \ + with the symbol, or drop it: {stale:?}" + ); + } + /// A mutating call, assembled at runtime. /// /// Spelling `env::set_var` literally in a fixture would make this file's own @@ -518,6 +573,15 @@ mod tests { format!("let _restore = crate::test_env::{path}(\"KEY\", &value);") } + /// A guard whose `Drop` mutates, armed by a test that never calls a + /// mutation itself. + /// + /// Assembled rather than written out for the same reason as + /// [`mutation_call`]. + fn held_mutating_guard(constructor: &str) -> String { + format!("let _held = {constructor}(&lock);") + } + fn unguarded_test(body: &str) -> String { format!( "#[cfg(test)]\nmod tests {{\n #[test]\n fn t() {{\n {body}\n }}\n}}\n" @@ -546,14 +610,15 @@ mod tests { /// `remove_var` inside a `#[test]` was flagged at all. /// /// The length assertion is what stops the next entry arriving untested: a - /// third call added to the const fails here until it gains a case. + /// new call added to the const fails here until it gains a case. #[test] fn every_mutating_call_is_flagged_inside_an_unguarded_test() { let kinds = ["set_var", "remove_var"]; let delegated = ["RestoredEnvVar::set"]; + let held = ["UnsetKeyOnExit::arm", "EnvVarGuard::set"]; assert_eq!( MUTATIONS.len(), - kinds.len() + delegated.len(), + kinds.len() + delegated.len() + held.len(), "a call was added to MUTATIONS without a case here" ); for kind in kinds { @@ -572,6 +637,14 @@ mod tests { "an unguarded {path} inside a #[test] must be flagged" ); } + for name in held { + let source = unguarded_test(&held_mutating_guard(name)); + assert_eq!( + env_mutations_in_unserialized_tests(&source).len(), + 1, + "a test holding an unguarded {name} must be flagged" + ); + } } /// A restoring wrapper does not exempt the test that drives it. @@ -601,6 +674,33 @@ mod tests { ); } + /// A guard whose `Drop` mutates does not exempt the test that holds it. + /// + /// The mirror of + /// [`a_mutation_delegated_to_a_restoring_wrapper_still_needs_the_lock`] + /// for a guard: the test body only calls the guard's constructor, so the + /// scan enforces the lock only while that call is in [`MUTATIONS`]. + /// Unguarded is an offense, guarded is not. + #[test] + fn holding_a_mutating_guard_still_needs_the_lock() { + let held = held_mutating_guard("UnsetKeyOnExit::arm"); + + let hits = env_mutations_in_unserialized_tests(&unguarded_test(&held)); + assert_eq!(hits.len(), 1, "holding the guard mutates on drop: {hits:?}"); + assert_eq!( + hits[0].call, "UnsetKeyOnExit::arm", + "the offender is named by the constructor call" + ); + + let guarded = unguarded_test(&format!( + "let lock = SOME_TEST_LOCK.lock().unwrap();\n {held}" + )); + assert!( + env_mutations_in_unserialized_tests(&guarded).is_empty(), + "the lock above is what makes this shape acceptable" + ); + } + /// The qualifier is a matter of how the file imports, not of what the call /// does. Keying the match on `env::` made a test that had written /// `use std::env::set_var` invisible to the scan while doing exactly the