From e656f9b83ee6869d52dab6cafc16e1ab84378cd7 Mon Sep 17 00:00:00 2001 From: HackingGate Date: Fri, 16 Oct 2026 21:08:28 +0900 Subject: [PATCH] Report behavioral evidence separately from enforcement installation --- README.md | 1 + docs/DESIGN.md | 18 ++- docs/REFERENCE.md | 62 +++++++- src/check.rs | 59 +++++++- src/main.rs | 22 ++- src/probe.rs | 360 ++++++++++++++++++++++++++++++++++++++++++++- tests/probe_cli.rs | 325 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 833 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8632ed7..8673212 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,7 @@ uphold init --owner OWNER --visibility public # a first policy, claims and hoo uphold scan # content rules over the tree uphold scan --text - # a commit message, release note, PR body uphold check # the claims in policy/upheld.toml still hold +uphold check --probe # add fresh behavior evidence without changing acceptance uphold check --coverage # which rules here carry a principle uphold rules --effective # every rule inheritance resolved to, and where each runs uphold guard --stage pre-push # the guards for that Git hook diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 9322b87..ec04b99 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -269,7 +269,23 @@ silence alone is insufficient evidence of a clean run. These are criteria for choosing a provider. The reconciler currently credits external hook ids from their installation and takes the claim's word for what they enforce; inspecting a provider's local rule definitions is still planned. -Probing is opt-in, not a prerequisite enforced on every claim. +Probing is opt-in, not a prerequisite enforced on every claim. Reconciliation +reports installation independently of behavior, which starts unmeasured. +`uphold check --probe` adds measurements from the same invocation without +changing the installation verdict. A bad-and-clean pair can demonstrate only +the selected manager's named hook at the captured repository revision; missing +controls, unread runs, timeouts and observed failures cannot demonstrate it. +A wrapper's result does not certify its individual engine rules or a same-named +hook supplied through another manager. + +Behavioral evidence is transient. The command captures a clean committed +snapshot before loading the policy and evaluating installation, runs that +exact revision, and refuses to certify changes during measurement. Provenance +includes the hook declaration, runner, stage and fixtures, so the exercised +subject is visible. This avoids a storage format or cache whose freshness a +later invocation would have to prove. External policies, machine dependencies +and remote services remain outside that snapshot; even a qualifying pair proves +only its exercised cases. ## Evidence: what a provider reports and what a policy may read diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 314b74c..124d8a3 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -3800,7 +3800,13 @@ when the result is refused or unavailable: "principle": "explicit-unknown", "rule": "catalog-tests", "status": "reconciled", - "suppliers": [{"component": "local", "seam": "a hook installed here"}] + "suppliers": [{"component": "local", "seam": "a hook installed here"}], + "behavior": [{ + "rule": "catalog-tests", "subject": "rule", + "manager": null, "stage": null, "source": null, + "state": "unmeasured", "outcome": "not_requested", + "detail": "no behavioral probe requested", "provenance": null + }] } ], "violations": [], @@ -3831,6 +3837,60 @@ A range-only hook configured exclusively at stages without a push range does not supply a claim. A valid declaration of the same id at another stage or through another runner still supplies it independently. +Installation and behavior are separate evidence dimensions. A reconciled claim +establishes installation from repository configuration. Each claim's `behavior` +array says whether a control was exercised: `unmeasured`, `demonstrated`, or +`measured_failure`. Ordinary checks report `unmeasured` with +`outcome = "not_requested"`; they need no runner or Git repository to do so. +The human report prints behavior separately from the installation evidence. + +```sh +uphold check --probe # installation plus fresh behavioral evidence +uphold check --probe --json # the same evidence as a structured report +``` + +`--probe` runs the file probes in `policy/hooks.toml` during this invocation, +using the runner detected from root configuration and PATH, as `uphold probe` +does. It adds evidence without changing reconciliation's exit status or making +probes a prerequisite for claims. A failed probe can therefore accompany a +clean installation verdict. Use `uphold probe` when the probe verdict itself +must determine the process exit. `--coverage` is a separate mode. + +A demonstrated measurement requires both refusal and acceptance controls. Its +subject is the exact hook id in the selected manager's root configuration at +the probed stage. The report names `manager`, `stage` and `source` separately; +a same-named hook under another manager remains unmeasured. A hook probe does +not certify an engine rule behind its wrapper. Empty-range push probes exercise +Git's delegate, so their labels are not matched to runner-hook claims. + +`cannot_fail`, `refuses_everything` and `unexpected_refusal` are measured +failures. Known startup, signal and snapshot-read failures during a control +are `unread` measured failures; an unread prerequisite leaves behavior +unmeasured. `timeout`, +`missing_clean_control`, `no_matching_probe` and `unmatched` remain unmeasured. +No missing probe, timeout or recognized unread condition establishes a +demonstration. Runner exit codes follow the existing probe contract: nonzero +for the bad control, zero for the clean control. Third-party runners have no +universal code for an internal error; declare `expect` to bind a nonzero exit +to the intended refusal rather than relying on the exit alone. + +Measured evidence carries `provenance`: the runner executable path, manager, +hook, stage, repository revision, configuration source, canonical hook +declaration, upstream and pin, and the fixture's path, refusal contents, +acceptance contents and expected refusal text. A clean committed snapshot is +captured before loading the policy and reconciling. Probes run a detached +checkout of that exact revision. Changes outside the fixture during either +control invalidate the pair. A dirty tree or a changed revision before or after +measurement leaves behavior unmeasured with the reason reported. + +No behavioral evidence is saved or reused: changing a rule, fixture or revision +requires a new run. Demonstration establishes only the exercised cases at that +snapshot, not universal correctness, coverage of every seam, or independence +from the author of the fixtures. The snapshot does not capture external +inherited policies, runner dependencies, machine configuration, or remote +services; the report is evidence of this invocation, not a promise of future +reproducibility. + The Python review and OSCAL consumers read this JSON contract. They reject unknown schema versions, malformed or inconsistent reports, unexpected process statuses, and any unavailable evidence. Review accepts an evaluated refusal diff --git a/src/check.rs b/src/check.rs index b350487..86b0934 100644 --- a/src/check.rs +++ b/src/check.rs @@ -765,6 +765,8 @@ pub(crate) struct EvaluatedClaim { pub(crate) rule: String, status: ClaimStatus, suppliers: Vec, + /// Independent observations; these never change the installation verdict. + behavior: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] @@ -782,7 +784,13 @@ struct Supplier { } /// `uphold check`, its JSON form, and the separate coverage report. -pub(crate) fn run(root: &Path, policy: &Policy, coverage: bool, json: bool) -> Result { +pub(crate) fn run( + root: &Path, + policy: &Policy, + coverage: bool, + json: bool, + probe_revision: Option>, +) -> Result { let path = root.join(DECLARATION); if !path.is_file() { return Err(Fatal::new(format!( @@ -799,7 +807,41 @@ pub(crate) fn run(root: &Path, policy: &Policy, coverage: bool, json: bool) -> R if coverage { return report_coverage(policy, &declaration, &installed, &supply); } - let report = evaluate(&path, &declaration, &installed, &supply)?; + let mut report = evaluate(&path, &declaration, &installed, &supply)?; + let measured = probe_revision + .map(|revision| revision.and_then(|revision| crate::probe::measure(root, &revision))); + for claim in &mut report.claims { + let (outcome, detail) = match &measured { + None => ("not_requested", "no behavioral probe requested".to_owned()), + Some(Err(error)) => ("unread", error.to_string()), + Some(Ok(_)) => ( + "no_matching_probe", + "no qualifying probe ran for this rule".to_owned(), + ), + }; + if let Some(Ok(evidence)) = &measured { + claim.behavior.extend( + evidence + .iter() + .filter(|item| item.rule == claim.rule) + .cloned(), + ); + } + // A wrapper hook's probe cannot certify the engine's separate rule, + // even when both happen to use the same identifier. + if claim.behavior.is_empty() + || claim + .suppliers + .iter() + .any(|supplier| supplier.component == "uphold") + { + claim.behavior.push(crate::probe::Evidence::unmeasured( + &claim.rule, + outcome, + detail, + )); + } + } if json { println!( "{}", @@ -935,6 +977,7 @@ fn evaluate( principle: principle.to_owned(), rule: rule.to_owned(), status, + behavior: Vec::new(), suppliers: if status == ClaimStatus::Reconciled { by.into_iter() .flatten() @@ -1006,6 +1049,18 @@ fn report_human(report: &Report) -> Result { .join(", ") ); } + for claim in &report.claims { + for evidence in &claim.behavior { + let manager = evidence + .manager + .as_ref() + .map_or(String::new(), |manager| format!(" ({manager})")); + println!( + " behavior {}{manager}: {} [{}] — {}", + claim.rule, evidence.state, evidence.outcome, evidence.detail + ); + } + } for note in &report.notes { println!(" note {note}"); } diff --git a/src/main.rs b/src/main.rs index 32e2531..a2a2d21 100644 --- a/src/main.rs +++ b/src/main.rs @@ -115,7 +115,7 @@ usage: uphold scan --text [FILE|-] run the host-identity rules over text uphold guard --stage STAGE run the guards that fire at STAGE uphold guard --text [FILE|-] run the text-capable guards over text - uphold check [--json] reconcile policy/upheld.toml against what runs + uphold check [--json] [--probe] reconcile claims; optionally measure hook behavior uphold check --coverage which rules run here and carry no principle uphold audit --for-publication what a private->public flip would republish uphold init --owner OWNER write a first policy, claims and hook config @@ -362,11 +362,18 @@ fn run() -> Result { "guard" => guard_command(rest), "audit" => audit_command(rest), "check" => match rest { - [] => check_command(false, false), - [flag] if flag == "--coverage" => check_command(true, false), - [flag] if flag == "--json" => check_command(false, true), + [] => check_command(false, false, false), + [flag] if flag == "--coverage" => check_command(true, false, false), + [flag] if flag == "--json" => check_command(false, true, false), + [flag] if flag == "--probe" => check_command(false, false, true), + [check_flag, other_flag] + if (check_flag == "--probe" && other_flag == "--json") + || (check_flag == "--json" && other_flag == "--probe") => + { + check_command(false, true, true) + } _ => Err(Fatal::new(format!( - "usage: uphold check [--coverage | --json]\n\n{USAGE}" + "usage: uphold check [--json] [--probe] | --coverage\n\n{USAGE}" ))), }, "hooks" => match rest { @@ -581,11 +588,12 @@ fn run() -> Result { /// The loader runs first and its answer is what the reconcile reads, which is /// the whole point of moving this in: `uphold_check.py` re-implemented /// `config::load` to answer the same question and was free to disagree with it. -fn check_command(coverage: bool, json: bool) -> Result { +fn check_command(coverage: bool, json: bool, probe: bool) -> Result { let working = std::env::current_dir()?; let (root, policy_path) = discover(&working).ok_or_else(|| no_policy_here(&working))?; + let probe_revision = probe.then(|| probe::snapshot(&root)); let policy = config::load(&root, &policy_path)?; - check::run(&root, &policy, coverage, json) + check::run(&root, &policy, coverage, json, probe_revision) } fn scan_command(arguments: &[OsString]) -> Result { diff --git a/src/probe.rs b/src/probe.rs index b262d57..6cc54f9 100644 --- a/src/probe.rs +++ b/src/probe.rs @@ -44,7 +44,7 @@ use std::collections::BTreeSet; use std::path::{Component, Path, PathBuf}; use std::process::Command; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::error::{Exit, Fatal, Result, read_to_string}; use crate::pins::{Manager, declarations}; @@ -140,6 +140,272 @@ enum Verdict { Unmeasured { after: u64 }, } +/// Behavior belongs to the named hook and runner, independently of installation. +/// Evidence is returned only from this invocation; it is never loaded from disk. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct Evidence { + /// The claim identifier this observation can be matched to. + pub rule: String, + /// `hook` bounds the result to one runner hook; `rule` is an unmeasured claim. + pub subject: &'static str, + /// Absent when no runner-specific subject was established. + pub manager: Option, + /// The Git stage exercised, so another stage cannot inherit the result. + pub stage: Option, + /// Repository-relative configuration defining the observed hook. + pub source: Option, + /// The evidence dimension, independent of installation and process status. + pub state: &'static str, + /// Stable reason code distinguishing failures and incomplete measurements. + pub outcome: &'static str, + /// Human explanation; consumers use outcome rather than parsing these words. + pub detail: String, + /// Present for an attempted control pair, including failed attempts. + pub provenance: Option, +} + +/// The repository snapshot and execution subject of one attempted control pair. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct Provenance { + /// Executable resolved on PATH for this invocation; external dependencies are not captured. + pub runner: String, + /// The configuration namespace the runner read. + pub manager: String, + /// The exact id requested from the runner in isolation. + pub hook: String, + /// The stage passed to that runner for both controls. + pub stage: String, + /// Committed revision captured before policy loading and checked after measurement. + pub revision: String, + /// The root configuration containing the matched declaration. + pub source: String, + /// Canonical hook body, retained whole so unknown runner fields remain visible. + pub declaration: String, + /// The declared origin of the hook when the configuration names one. + pub upstream: Option, + /// The upstream revision declared in the runner configuration. + pub pin: Option, + /// Both controls and the refusal text used to judge their outcomes. + pub fixture: Fixture, +} + +/// Reviewable controls copied from the committed probe declaration. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct Fixture { + /// Repository-relative file planted in the disposable checkout. + pub path: Option, + /// Bad control contents the hook must reject. + pub refuses: String, + /// Clean control contents; absence cannot establish demonstration. + pub allows: Option, + /// Required refusal text when the fixture declares a more specific reason. + pub expect: Option, +} + +impl Evidence { + /// Explain absence of behavioral proof without inventing a measured subject. + pub(crate) fn unmeasured(rule: &str, outcome: &'static str, detail: String) -> Self { + Self { + rule: rule.to_owned(), + subject: "rule", + manager: None, + stage: None, + source: None, + state: "unmeasured", + outcome, + detail, + provenance: None, + } + } +} + +/// Drive the root configuration's hooks at one clean, immutable revision. +/// +/// A reconciliation reads the working tree, while a probe runs a checkout. A +/// dirty tree cannot identify those two as the same subject. Refuse to certify +/// it, and check again afterwards so a concurrent edit cannot inherit evidence. +/// External programs and services are not captured by this repository snapshot. +pub(crate) fn measure(root: &Path, revision: &str) -> Result> { + if snapshot(root)? != revision { + return Err(Fatal::new( + "repository revision changed after reconciliation began; behavioral evidence would describe a different configuration", + )); + } + let (probes, file_timeout) = probes(root)?; + let declarations = declarations(root)?; + if probes.is_empty() { + return Ok(Vec::new()); + } + let runner = detect(root)?; + let mut evidence = Vec::new(); + for probe in &probes { + // A push probe exercises git's installed delegate, not a runner hook + // whose arbitrary label happens to equal an enforcement claim. + if probe.push.is_some() { + continue; + } + let hook_stage = probe.stage.as_deref().unwrap_or("pre-commit"); + let matching: Vec<_> = declarations + .iter() + .filter(|declaration| { + declaration.id == probe.id + && declaration.manager == runner.manager() + && Path::new(&declaration.source).parent() == Some(Path::new("")) + && declaration + .stage + .as_deref() + .is_none_or(|declared| declared == hook_stage) + }) + .collect(); + let [declaration] = matching.as_slice() else { + evidence.push(Evidence::unmeasured( + &probe.id, + "unmatched", + format!( + "the selected {} root configuration has {} matching declarations at {hook_stage}", + runner.command(), + matching.len() + ), + )); + continue; + }; + let worktree = Worktree::at_revision(root, revision)?; + let result = drive( + &worktree, + runner, + probe, + probe.timeout_seconds.or(file_timeout), + true, + ); + let (state, outcome, detail) = match result { + Ok(Verdict::Demonstrated) => ( + "demonstrated", + "bad_and_clean", + "refuses its fixture, accepts a clean one".to_owned(), + ), + Ok(Verdict::RefusedOnly) => ( + "unmeasured", + "missing_clean_control", + "no allows fixture establishes acceptance".to_owned(), + ), + Ok(Verdict::CannotFail) => ( + "measured_failure", + "cannot_fail", + "accepted the fixture it must refuse".to_owned(), + ), + Ok(Verdict::RefusesEverything) => ( + "measured_failure", + "refuses_everything", + "refused the clean fixture as well".to_owned(), + ), + Ok(Verdict::RefusedForAnotherReason { said }) => { + ("measured_failure", "unexpected_refusal", said) + } + Ok(Verdict::Unmeasured { after }) => ( + "unmeasured", + "timeout", + format!("no verdict after {after}s"), + ), + Err(error) => ("measured_failure", "unread", error.to_string()), + }; + evidence.push(Evidence { + rule: probe.id.clone(), + subject: "hook", + manager: Some(runner.manager().as_str().to_owned()), + stage: Some(hook_stage.to_owned()), + source: Some(declaration.source.clone()), + state, + outcome, + detail, + provenance: Some(Provenance { + runner: which(runner.command()).map_or_else( + || runner.command().to_owned(), + |path| path.display().to_string(), + ), + manager: runner.manager().as_str().to_owned(), + hook: probe.id.clone(), + stage: hook_stage.to_owned(), + revision: revision.to_owned(), + source: declaration.source.clone(), + declaration: declaration.body.clone(), + upstream: declaration.from.clone(), + pin: declaration.rev.clone(), + fixture: Fixture { + path: probe.path.clone(), + refuses: probe.refuses.clone(), + allows: probe.allows.clone(), + expect: probe.expect.clone(), + }, + }), + }); + } + for declaration in &declarations { + if declaration.manager == Manager::Git + || Path::new(&declaration.source).parent() != Some(Path::new("")) + || evidence.iter().any(|item| { + item.rule == declaration.id + && item.manager.as_deref() == Some(declaration.manager.as_str()) + && item.source.as_deref() == Some(declaration.source.as_str()) + && declaration + .stage + .as_deref() + .is_none_or(|stage| item.stage.as_deref() == Some(stage)) + }) + { + continue; + } + let mut item = Evidence::unmeasured( + &declaration.id, + "no_matching_probe", + "no qualifying probe ran for this hook declaration".to_owned(), + ); + item.subject = "hook"; + item.manager = Some(declaration.manager.as_str().to_owned()); + item.stage.clone_from(&declaration.stage); + item.source = Some(declaration.source.clone()); + evidence.push(item); + } + if snapshot(root)? != revision { + return Err(Fatal::new( + "repository revision changed while probing; behavioral evidence is stale", + )); + } + Ok(evidence) +} + +/// Identify a committed subject only when the working tree agrees with it. +pub(crate) fn snapshot(root: &Path) -> Result { + let status = detached("git", root) + .args([ + "status", + "--porcelain", + "--untracked-files=all", + "--ignore-submodules=none", + ]) + .output() + .map_err(|error| Fatal::new(format!("could not establish probe snapshot: {error}")))?; + if !status.status.success() { + return Err(Fatal::new(format!( + "could not establish probe snapshot: {}", + String::from_utf8_lossy(&status.stderr).trim() + ))); + } + if !status.stdout.is_empty() { + return Err(Fatal::new( + "behavioral evidence requires a clean committed tree; changed rules or fixtures cannot be certified by a probe of HEAD", + )); + } + let head = detached("git", root) + .args(["rev-parse", "--verify", "HEAD"]) + .output()?; + if !head.status.success() { + return Err(Fatal::new( + "could not identify the committed revision to probe", + )); + } + Ok(String::from_utf8_lossy(&head.stdout).trim().to_owned()) +} + /// `uphold probe` pub(crate) fn run(root: &Path, runner: Option<&str>, timeout: Option) -> Result { let (probes, file_timeout) = probes(root)?; @@ -200,7 +466,7 @@ pub(crate) fn run(root: &Path, runner: Option<&str>, timeout: Option) -> Re } else { probe.id.clone() }; - results.push((label, drive(&worktree, runner, probe, patience)?)); + results.push((label, drive(&worktree, runner, probe, patience, false)?)); } drop(worktree); @@ -282,6 +548,7 @@ fn drive( runner: Runner, probe: &Probe, patience: Option, + verify_snapshot: bool, ) -> Result { let stage = probe.stage.as_deref().unwrap_or("pre-commit"); let run = |fixture: &str| -> Result { @@ -293,6 +560,9 @@ fn drive( let path = probe.path.as_deref().unwrap_or_default(); worktree.plant(path, fixture)?; let ran = runner.drive(worktree.path(), &probe.id, path, stage, patience); + if verify_snapshot { + worktree.unchanged_except(path)?; + } worktree.remove(path)?; ran }; @@ -588,16 +858,21 @@ pub(crate) fn detached(program: &str, directory: &Path) -> Command { struct Worktree { root: PathBuf, path: PathBuf, + revision: String, } impl Worktree { fn at(root: &Path) -> Result { + Self::at_revision(root, "HEAD") + } + + fn at_revision(root: &Path, revision: &str) -> Result { let path = std::env::temp_dir().join(format!("uphold-probe-{}", std::process::id())); drop(std::fs::remove_dir_all(&path)); let output = detached("git", root) .args(["worktree", "add", "--detach", "--quiet"]) .arg(&path) - .arg("HEAD") + .arg(revision) .output() .map_err(|error| Fatal::new(format!("could not add a worktree: {error}")))?; if !output.status.success() { @@ -609,6 +884,7 @@ impl Worktree { Ok(Self { root: root.to_path_buf(), path, + revision: revision.to_owned(), }) } @@ -616,6 +892,34 @@ impl Worktree { &self.path } + fn unchanged_except(&self, fixture: &str) -> Result<()> { + let head = detached("git", &self.path) + .args(["rev-parse", "--verify", "HEAD"]) + .output()?; + if !head.status.success() || String::from_utf8_lossy(&head.stdout).trim() != self.revision { + return Err(Fatal::new( + "hook changed the probe revision; the controls did not measure the same committed configuration", + )); + } + let status = detached("git", &self.path) + .args([ + "status", + "--porcelain", + "--untracked-files=all", + "--ignore-submodules=none", + "--", + ".", + ]) + .arg(format!(":(exclude,literal){fixture}")) + .output()?; + if !status.status.success() || !status.stdout.is_empty() { + return Err(Fatal::new( + "hook changed the probe snapshot outside its fixture; the controls did not measure the same committed configuration", + )); + } + Ok(()) + } + fn plant(&self, relative: &str, contents: &str) -> Result<()> { let file = self.fixture_path(relative)?; if let Some(parent) = file.parent() { @@ -945,6 +1249,56 @@ mod tests { } } + #[test] + fn a_revision_captured_before_reconciliation_must_still_match_before_measurement() { + let root = crate::fixture::scratch("probe-snapshot"); + std::fs::create_dir_all(&root).unwrap(); + for arguments in [ + vec!["init", "--quiet"], + vec!["config", "user.name", "Test"], + vec!["config", "user.email", "test@example.test"], + vec![ + "-c", + "core.hooksPath=/dev/null", + "commit", + "--allow-empty", + "--quiet", + "-m", + "first", + ], + ] { + assert!( + detached("git", &root) + .args(arguments) + .status() + .unwrap() + .success() + ); + } + let captured = snapshot(&root).unwrap(); + assert!( + detached("git", &root) + .args([ + "-c", + "core.hooksPath=/dev/null", + "commit", + "--allow-empty", + "--quiet", + "-m", + "advanced" + ]) + .status() + .unwrap() + .success() + ); + let error = measure(&root, &captured).unwrap_err(); + assert!( + error + .to_string() + .contains("revision changed after reconciliation began") + ); + } + #[test] fn a_runner_is_named_by_the_command_it_runs() { // The mapping is read in two directions -- a name from the command line diff --git a/tests/probe_cli.rs b/tests/probe_cli.rs index a5d4b34..f398cb3 100644 --- a/tests/probe_cli.rs +++ b/tests/probe_cli.rs @@ -704,3 +704,328 @@ fn a_runner_probe_with_no_path_is_refused_at_load() { assert_eq!(code(&output), 2, "{}", text(&output)); assert!(text(&output).contains("has no `path`"), "{}", text(&output)); } + +fn claimed_repository(probes: &str) -> PathBuf { + let root = repository(probes); + std::fs::write( + root.join("policy/principles.toml"), + "allowed_scripts = ['Latin']\n", + ) + .unwrap(); + std::fs::write( + root.join("policy/upheld.toml"), + "[[enforce]]\nprinciple = 'explicit-unknown'\nrule = 'no-markers'\n", + ) + .unwrap(); + support::git(&root, &["add", "-A"]); + support::git(&root, &["commit", "-qm", "claim", "--no-verify"]); + root +} + +fn reconciliation(root: &Path, stub: &Path, measure: bool) -> serde_json::Value { + let mut command = Command::new(env!("CARGO_BIN_EXE_uphold")); + command.args(["check", "--json"]); + if measure { + command.arg("--probe"); + } + let mut path = stub.as_os_str().to_owned(); + path.push(":/usr/bin:/bin"); + let output = command + .current_dir(root) + .env("PATH", path) + .stdin(Stdio::null()) + .output() + .unwrap(); + // Behavioral reporting is independent of the installation contract. + assert_eq!(code(&output), 0, "{}", text(&output)); + serde_json::from_slice(&output.stdout).unwrap() +} + +#[test] +fn installed_hook_with_no_probe_has_explicitly_unmeasured_behavior() { + let root = claimed_repository(""); + let stub = runner("exit 0"); + let measured = reconciliation(&root, &stub, true); + assert_eq!( + measured.pointer("/claims/0/behavior/0/state").unwrap(), + "unmeasured" + ); + assert_eq!( + measured.pointer("/claims/0/behavior/0/outcome").unwrap(), + "no_matching_probe" + ); + // Ordinary check needs neither a runner nor a git snapshot. + std::fs::remove_dir_all(root.join(".git")).unwrap(); + let report = reconciliation(&root, &stub, false); + let claim = &report.pointer("/claims/0").unwrap(); + assert_eq!(claim["status"], "reconciled"); + assert_eq!(claim.pointer("/behavior/0").unwrap()["state"], "unmeasured"); + assert_eq!( + claim.pointer("/behavior/0").unwrap()["outcome"], + "not_requested" + ); + assert!(claim.pointer("/behavior/0").unwrap()["provenance"].is_null()); +} + +#[test] +fn combined_report_demonstrates_only_the_matching_hook_with_provenance() { + let root = claimed_repository(ONE_PROBE); + let stub = runner("grep -q MARKER sample.txt && exit 1\nexit 0"); + let report = reconciliation(&root, &stub, true); + let behavior = &report.pointer("/claims/0/behavior/0").unwrap(); + assert_eq!(behavior["state"], "demonstrated"); + assert_eq!(behavior["subject"], "hook"); + assert_eq!(behavior["manager"], "pre-commit"); + let provenance = &behavior["provenance"]; + assert_eq!(provenance["hook"], "no-markers"); + assert_eq!(provenance["stage"], "pre-commit"); + assert_eq!(provenance["source"], ".pre-commit-config.yaml"); + assert_eq!(provenance.pointer("/fixture/refuses").unwrap(), "MARKER\n"); + assert_eq!(provenance.pointer("/fixture/allows").unwrap(), "clean\n"); + assert!(provenance["runner"].as_str().unwrap().ends_with("/prek")); + let head = support::git_command(&root) + .args(["rev-parse", "HEAD"]) + .output() + .unwrap(); + assert_eq!( + provenance["revision"], + String::from_utf8_lossy(&head.stdout).trim() + ); + assert!( + provenance["declaration"] + .as_str() + .unwrap() + .contains("no-markers") + ); +} + +#[test] +fn failed_and_partial_controls_never_demonstrate_a_claim_or_change_installation_acceptance() { + for (script, probes, state, outcome) in [ + ( + "exit 0", + ONE_PROBE.to_owned(), + "measured_failure", + "cannot_fail", + ), + ( + "exit 1", + ONE_PROBE.to_owned(), + "measured_failure", + "refuses_everything", + ), + ( + "kill -TERM $$", + ONE_PROBE.to_owned(), + "measured_failure", + "unread", + ), + ( + "sleep 2", + format!("timeout_seconds = 1\n{ONE_PROBE}"), + "unmeasured", + "timeout", + ), + ( + "exit 1", + ONE_PROBE.replace("allows = \"clean\\n\"", ""), + "unmeasured", + "missing_clean_control", + ), + ] { + let root = claimed_repository(&probes); + let stub = runner(script); + let report = reconciliation(&root, &stub, true); + let behavior = &report.pointer("/claims/0/behavior/0").unwrap(); + assert_eq!(behavior["state"], state, "{report}"); + assert_eq!(behavior["outcome"], outcome, "{report}"); + assert_eq!(report.pointer("/claims/0").unwrap()["status"], "reconciled"); + } +} + +#[test] +fn changed_rules_fixtures_and_revisions_cannot_reuse_demonstration() { + let root = claimed_repository(ONE_PROBE); + let stub = runner("grep -q MARKER sample.txt && exit 1\nexit 0"); + let first = reconciliation(&root, &stub, true); + let old_revision = first + .pointer("/claims/0/behavior/0/provenance/revision") + .unwrap() + .clone(); + for relative in [".pre-commit-config.yaml", "policy/hooks.toml"] { + let contents = std::fs::read_to_string(root.join(relative)).unwrap(); + std::fs::write(root.join(relative), format!("{contents}\n# changed\n")).unwrap(); + let report = reconciliation(&root, &stub, true); + assert_eq!( + report.pointer("/claims/0/behavior/0").unwrap()["state"], + "unmeasured" + ); + assert!( + report.pointer("/claims/0/behavior/0").unwrap()["detail"] + .as_str() + .unwrap() + .contains("clean committed tree") + ); + std::fs::write(root.join(relative), contents).unwrap(); + } + std::fs::write( + root.join("policy/hooks.toml"), + ONE_PROBE.replace("MARKER", "OTHER"), + ) + .unwrap(); + support::git(&root, &["add", "-A"]); + support::git(&root, &["commit", "-qm", "changed fixture", "--no-verify"]); + let changed = reconciliation(&root, &stub, true); + let behavior = &changed.pointer("/claims/0/behavior/0").unwrap(); + assert_eq!(behavior["state"], "measured_failure"); + assert_eq!(behavior["outcome"], "cannot_fail"); + assert_ne!( + behavior.pointer("/provenance/revision").unwrap(), + &old_revision + ); + assert_eq!( + behavior.pointer("/provenance/fixture/refuses").unwrap(), + "OTHER\n" + ); +} + +#[test] +fn mutation_between_controls_is_unread_instead_of_demonstrated() { + let root = claimed_repository(ONE_PROBE); + let stub = runner( + "echo '# changed by hook' >> .pre-commit-config.yaml\ngrep -q MARKER sample.txt && exit 1\nexit 0", + ); + let report = reconciliation(&root, &stub, true); + let behavior = &report.pointer("/claims/0/behavior/0").unwrap(); + assert_eq!(behavior["state"], "measured_failure"); + assert_eq!(behavior["outcome"], "unread"); + assert!( + behavior["detail"] + .as_str() + .unwrap() + .contains("changed the probe snapshot") + ); +} + +#[test] +fn same_id_in_another_manager_does_not_inherit_a_demonstration() { + let root = claimed_repository(ONE_PROBE); + std::fs::write( + root.join("lefthook.yml"), + "pre-commit:\n commands:\n no-markers:\n run: echo unrelated\n", + ) + .unwrap(); + support::git(&root, &["add", "-A"]); + support::git(&root, &["commit", "-qm", "second manager", "--no-verify"]); + let stub = runner("grep -q MARKER sample.txt && exit 1\nexit 0"); + let report = reconciliation(&root, &stub, true); + let measurements = report + .pointer("/claims/0/behavior") + .unwrap() + .as_array() + .unwrap(); + assert_eq!(measurements.len(), 2, "{report}"); + assert!( + measurements + .iter() + .any(|item| item["manager"] == "pre-commit" && item["state"] == "demonstrated") + ); + assert!( + measurements + .iter() + .any(|item| item["manager"] == "lefthook" && item["state"] == "unmeasured") + ); +} + +#[test] +fn a_push_probe_label_cannot_certify_a_runner_hook() { + let root = claimed_repository( + "[[probe]]\nid = 'no-markers'\npush = 'empty'\nrefuses = 'other/repo'\nallows = 'example/repo'\n", + ); + let stub = runner("exit 1"); + let report = reconciliation(&root, &stub, true); + let evidence = report.pointer("/claims/0/behavior/0").unwrap(); + assert_eq!(evidence["state"], "unmeasured"); + assert_eq!(evidence["outcome"], "no_matching_probe"); +} + +#[test] +fn a_wrapper_hook_does_not_certify_an_engine_rule_with_the_same_name() { + let root = claimed_repository(ONE_PROBE); + std::fs::write(root.join("policy/principles.toml"), "[rule.no-markers]\nregexp = 'MARKER'\nmessage = 'marker'\nfiles.include = ['sample.txt']\n").unwrap(); + let config = std::fs::read_to_string(root.join(".pre-commit-config.yaml")).unwrap(); + std::fs::write(root.join(".pre-commit-config.yaml"), format!("{config} - repo: https://github.com/HackingGate/uphold\n rev: v1.0.0\n hooks:\n - id: uphold-scan\n")).unwrap(); + support::git(&root, &["add", "-A"]); + support::git(&root, &["commit", "-qm", "engine supplier", "--no-verify"]); + let stub = runner("grep -q MARKER sample.txt && exit 1\nexit 0"); + let report = reconciliation(&root, &stub, true); + let evidence = report + .pointer("/claims/0/behavior") + .unwrap() + .as_array() + .unwrap(); + assert!( + evidence + .iter() + .any(|item| item["subject"] == "hook" && item["state"] == "demonstrated"), + "{report}" + ); + assert!( + evidence + .iter() + .any(|item| item["subject"] == "rule" && item["state"] == "unmeasured"), + "{report}" + ); +} + +#[test] +fn a_revision_changed_during_the_probe_invalidates_all_demonstrations() { + let root = claimed_repository(ONE_PROBE); + // Synthetic local fixture commits are the test subject; this never changes + // the checkout running the suite. + let script = format!( + "git -C '{}' -c core.hooksPath=/dev/null commit --allow-empty -qm advanced\ngrep -q MARKER sample.txt && exit 1\nexit 0", + root.display() + ); + let stub = runner(&script); + let report = reconciliation(&root, &stub, true); + let evidence = report.pointer("/claims/0/behavior/0").unwrap(); + assert_eq!(evidence["state"], "unmeasured"); + assert!( + evidence["detail"] + .as_str() + .unwrap() + .contains("revision changed") + ); +} + +#[test] +fn human_reconciliation_names_behavior_separately_from_installation() { + let root = claimed_repository(""); + let output = Command::new(env!("CARGO_BIN_EXE_uphold")) + .arg("check") + .current_dir(&root) + .output() + .unwrap(); + assert_eq!(code(&output), 0, "{}", text(&output)); + assert!(text(&output).contains("enforced by a hook installed here")); + assert!(text(&output).contains("behavior no-markers: unmeasured")); +} + +#[test] +fn committing_a_changed_probe_snapshot_cannot_hide_mutation_between_controls() { + let root = claimed_repository(ONE_PROBE); + let stub = runner( + "echo '# changed by hook' >> .pre-commit-config.yaml\ngit add -A\ngit -c core.hooksPath=/dev/null commit -qm changed\ngrep -q MARKER sample.txt && exit 1\nexit 0", + ); + let report = reconciliation(&root, &stub, true); + let evidence = report.pointer("/claims/0/behavior/0").unwrap(); + assert_eq!(evidence["state"], "measured_failure"); + assert_eq!(evidence["outcome"], "unread"); + assert!( + evidence["detail"] + .as_str() + .unwrap() + .contains("changed the probe revision") + ); +}