diff --git a/docs/experiments/bdv-pilot/README.md b/docs/experiments/bdv-pilot/README.md new file mode 100644 index 00000000..ba65c82f --- /dev/null +++ b/docs/experiments/bdv-pilot/README.md @@ -0,0 +1,34 @@ +# BDV pilot: approval call binding + +This pilot links a frozen behavior claim to a retained integration probe and a machine-readable evidence ledger. It does not change product behavior or replace existing tests. + +## Run + +```sh +pnpm bdv approval-call-binding +``` + +The runner requires a clean checkout (the evidence ledger itself may already contain earlier runs) and checks that the probe matches the digest frozen in [`claims.json`](claims.json). It then runs Vitest's JSON reporter with a 120-second process timeout and requires each of the four exact scenario names to appear exactly once with `passed` status. Aggregate counts or earlier console text cannot produce a passing verdict. + +Before and after the probe, it records the commit and hashes the claim, probe, and runner. It rejects dirty executable inputs and any change to those hashes or `HEAD` during execution. The appended ledger entry includes before/after input hashes, the command, timeout and exit status, exact scenario results, result digest, elapsed time, output digest, and normalized output. This makes the observed result checkable against the committed contract and probe. + +## Independence and limits + +Acceptance is a fixed contract that the runner does not generate from the observed output. It expresses the approval rule as externally observable outcomes: only the allowed file is created, the denied target survives, each tool call receives its own decision, and the provider sees paired calls and results. The probe crosses the built CLI, a loopback HTTP provider fixture, and isolated filesystem effects. Its approved source digest and exact scenario names bind the runner to the reviewed probe; changes require an explicit contract update. + +This is one claim and one probe, not a general probe scheduler. The claim contract and its probe are maintained in the same repository, so hashes establish provenance and change detection, not independent authorship. An independent reviewer has not yet challenged this pilot contract. It does not yet enforce independent contract approval, compare costs across broad test suites, or show that passing the selected scenarios proves all approval paths safe. + +## Baseline and pilot result + +Baseline at `d93142ea` (before the runner), using the same probe directly: + +```text +pnpm exec vitest run --reporter=minimal --config vitest.integration.config.ts source/cli.integration.test.ts -t 'an approved shell call does not approve a later call' +Test Files 1 passed (1) +Tests 4 passed | 30 skipped (34) +Duration 2.60s +``` + +The baseline establishes repeatability and elapsed test time only; it is not a full-project defect-detection or cost comparison. The pilot evidence record is appended when `pnpm bdv approval-call-binding` runs. + +The first runner execution result and elapsed time are captured in the evidence ledger. Compare them with the baseline as a one-run observation only; this pilot does not establish a performance trend. diff --git a/docs/experiments/bdv-pilot/claims.json b/docs/experiments/bdv-pilot/claims.json new file mode 100644 index 00000000..d83147dd --- /dev/null +++ b/docs/experiments/bdv-pilot/claims.json @@ -0,0 +1,27 @@ +{ + "schemaVersion": 1, + "claims": [ + { + "id": "approval-call-binding", + "statement": "An approval for one shell tool call cannot authorize a later call that reuses the same provider call id, and adapter-generated ids for missing or empty provider ids remain distinct.", + "authority": "Frozen pilot contract derived from the existing user-facing approval policy and the expected safe side effects; this runner cannot alter the expected outcomes.", + "probe": "approval-reuse-integration", + "approvedProbeDigest": "3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061", + "acceptance": { + "expectedCases": [ + "an approved shell call does not approve a later call that reuses its id when the provider repeats call_0", + "an approved shell call does not approve a later call that reuses its id when the provider repeats its own id", + "an approved shell call does not approve a later call when the chat-completions provider omits tool call ids", + "an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids" + ], + "mustAssert": [ + "the first GREEN unsandboxed command runs and writes exactly 'approved\\n'", + "the subsequent RED rm command is refused and its victim file remains exactly 'must survive\\n'", + "the CLI reports two distinct approval decisions and the RED refusal reason", + "all four provider-id, repeated-id, omitted-id, and empty-id scenarios pass", + "the provider receives both calls and paired tool results" + ] + } + } + ] +} diff --git a/docs/experiments/bdv-pilot/evidence.jsonl b/docs/experiments/bdv-pilot/evidence.jsonl new file mode 100644 index 00000000..38e236e6 --- /dev/null +++ b/docs/experiments/bdv-pilot/evidence.jsonl @@ -0,0 +1,7 @@ +{"schemaVersion":1,"claimId":"approval-call-binding","claimDigest":"aa52df0624b9888e789272e09bf21d8126663b4802664671afcfffb3788d5d07","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigest":"c8ed6477c63fc2507a7979a6c92db43c89d19c8f48003465626d67808d2e0be6","commit":"0425b5a12e6b4429f3621d16a6f3c38205e0dc5e","startedAt":"2026-10-08T15:45:59.677Z","durationMs":2900,"command":["pnpm","exec","vitest","run","--reporter=minimal","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call"],"exitCode":0,"passedFiles":1,"passedTests":4,"verdict":"pass","outputDigest":"b89ec2955b9af86381110d1be4d22669b178c74c2cc45f149e87728f4a71e16d","output":"\n RUN v4.1.9 \n\n\n Test Files 1 passed (1)\n Tests 4 passed | 30 skipped (34)\n Start at 22:45:59\n Duration 2.59s (transform 31ms, setup 12ms, import 36ms, tests 2.48s, environment 0ms)\n\n"} +{"schemaVersion":2,"claimId":"approval-call-binding","claimDigest":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","approvedProbeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigest":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","commit":"47ccac4811b79e2bba92d7d87123568718f0d227","commitAfter":"47ccac4811b79e2bba92d7d87123568718f0d227","claimDigestAfter":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","probeDigestAfter":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigestAfter":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","inputsStable":true,"dirtyPathsBefore":[],"dirtyPathsAfter":[],"startedAt":"2026-10-08T16:00:07.451Z","durationMs":3002,"timeoutMs":120000,"timedOut":false,"command":["pnpm","exec","vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/vitest-report.json"],"exitCode":0,"totalTests":34,"passedTests":4,"failedTests":0,"scenarioResults":[{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","status":"passed"},{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","status":"passed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","status":"passed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids","status":"passed"}],"resultDigest":"327c214cadb80219c3d2b660ea5200e344705cef172b90a92e2e2644206861f7","verdict":"pass","outputDigest":"3dfac86c28083d2005ef7131a28dcc65fe54a0bd51fce1c6c61a431bea5a2fa7","output":"JSON report written to /vitest-report.json\n","error":null} +{"schemaVersion":2,"claimId":"approval-call-binding","claimDigest":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","approvedProbeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigest":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","commit":"47ccac4811b79e2bba92d7d87123568718f0d227","commitAfter":"47ccac4811b79e2bba92d7d87123568718f0d227","claimDigestAfter":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","probeDigestAfter":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigestAfter":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","inputsStable":false,"dirtyPathsBefore":[".bdv-dirty-input-probe"],"dirtyPathsAfter":[".bdv-dirty-input-probe"],"startedAt":"2026-10-08T16:00:40.098Z","durationMs":39,"timeoutMs":120000,"timedOut":false,"command":["pnpm","exec","vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/vitest-report.json"],"exitCode":1,"totalTests":null,"passedTests":null,"failedTests":null,"scenarioResults":[{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","status":"missing"},{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","status":"missing"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","status":"missing"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids","status":"missing"}],"resultDigest":"4e05a53322a157c8f07503e5ab538b4e253fb931fefd66609e547aa7b3c8ba52","verdict":"fail","outputDigest":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","output":"","error":null} +{"schemaVersion":2,"recordType":"seeded-fault-control","claimId":"approval-call-binding","baseCommit":"6ccef47824af7b3387c651c04e43395bb088d8f5","approvedProbeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","fault":"ApprovalLedger.blanketDecision treats any nonempty per-call approved list as blanket approval","faultPatchDigest":"14dfd2e9ed15f1a2e48711e6365f674779901bd71284018a2c38ceff616fa300","command":["CI=1","node_modules/.bin/vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/private/tmp/bdv-seeded-fault-report.json"],"exitCode":1,"totalTests":34,"passedTests":0,"failedTests":4,"failingScenarios":["an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids"],"observedFailure":"all four scenarios removed victim.txt instead of preserving its contents; assertion expected 'must survive\\n' but received null","reportDigest":"1d42575f8391b62fb5dd751f9cb5a21952dac3b565986794f15890dda824eb7c","verdict":"expected-fail"} +{"schemaVersion":2,"claimId":"approval-call-binding","claimDigest":"27afbbe942d3b4da4896418f7114ab707abdbc3647f85343daed90f3d243e82e","approvedProbeDigest":"c587fc30c0efe640d83a89f2ece92771406673c67102beca40ce84b3db3afb7c","probeDigest":"c587fc30c0efe640d83a89f2ece92771406673c67102beca40ce84b3db3afb7c","runnerDigest":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","commit":"1c51842882cbd2ea1d2ca828913a34a39776919d","commitAfter":"1c51842882cbd2ea1d2ca828913a34a39776919d","claimDigestAfter":"27afbbe942d3b4da4896418f7114ab707abdbc3647f85343daed90f3d243e82e","probeDigestAfter":"c587fc30c0efe640d83a89f2ece92771406673c67102beca40ce84b3db3afb7c","runnerDigestAfter":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","inputsStable":true,"dirtyPathsBefore":[],"dirtyPathsAfter":[],"startedAt":"2026-10-09T00:32:22.912Z","durationMs":3440,"timeoutMs":120000,"timedOut":false,"command":["pnpm","exec","vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/vitest-report.json"],"exitCode":1,"totalTests":37,"passedTests":0,"failedTests":4,"scenarioResults":[{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","status":"failed"},{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","status":"failed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","status":"failed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids","status":"failed"}],"resultDigest":"ecd2b6913729ad3adaf2786819612bcffffb7c6dc968518fabf19400d8974374","verdict":"fail","outputDigest":"3dfac86c28083d2005ef7131a28dcc65fe54a0bd51fce1c6c61a431bea5a2fa7","output":"JSON report written to /vitest-report.json\n","error":null} +{"schemaVersion":2,"claimId":"approval-call-binding","claimDigest":"be535d04c03fbea903ffa6542e20f005b2173249043201fcdb4acd0986eade07","approvedProbeDigest":"3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061","probeDigest":"3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061","runnerDigest":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","commit":"ee0d5cda571f23e47d721486fa4c2b20cc1c00af","commitAfter":"ee0d5cda571f23e47d721486fa4c2b20cc1c00af","claimDigestAfter":"be535d04c03fbea903ffa6542e20f005b2173249043201fcdb4acd0986eade07","probeDigestAfter":"3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061","runnerDigestAfter":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","inputsStable":true,"dirtyPathsBefore":[],"dirtyPathsAfter":[],"startedAt":"2026-10-09T00:33:36.674Z","durationMs":2955,"timeoutMs":120000,"timedOut":false,"command":["pnpm","exec","vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/vitest-report.json"],"exitCode":0,"totalTests":37,"passedTests":4,"failedTests":0,"scenarioResults":[{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","status":"passed"},{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","status":"passed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","status":"passed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids","status":"passed"}],"resultDigest":"25bb51501ca6c75f14849069f72d031eccc40cdddf499c9a54846dc1582a29a0","verdict":"pass","outputDigest":"3dfac86c28083d2005ef7131a28dcc65fe54a0bd51fce1c6c61a431bea5a2fa7","output":"JSON report written to /vitest-report.json\n","error":null} +{"schemaVersion":2,"recordType":"seeded-fault-control","claimId":"approval-call-binding","baseCommit":"ee0d5cda571f23e47d721486fa4c2b20cc1c00af","approvedProbeDigest":"3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061","probeDigest":"3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061","fault":"ApprovalLedger.blanketDecision treats any nonempty per-call approved list as blanket approval","faultPatchDigest":"14dfd2e9ed15f1a2e48711e6365f674779901bd71284018a2c38ceff616fa300","command":["CI=1","node_modules/.bin/vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/private/tmp/bdv-rebased-seeded.json"],"exitCode":1,"totalTests":37,"passedTests":0,"failedTests":4,"failingScenarios":["an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids"],"observedFailure":"each scenario deleted victim.txt instead of preserving 'must survive\\n'","reportDigest":"04f33822254120c6bb8248c36c2e7c3d2fbb20dea7ec83756d84dd1ed893aa8e","verdict":"expected-fail"} diff --git a/docs/experiments/evidence-driven-development/tasks.jsonl b/docs/experiments/evidence-driven-development/tasks.jsonl index aac458a9..f40f51ad 100644 --- a/docs/experiments/evidence-driven-development/tasks.jsonl +++ b/docs/experiments/evidence-driven-development/tasks.jsonl @@ -1,2 +1,3 @@ {"id":"2026-10-08-approval-reuse-guard","type":"infra","risk":"high","started_at":"2026-10-08T20:52:58+07:00","verified_at":"2026-10-08T21:00:53+07:00","commit":"6ccef478","files_changed":1,"lines_changed":166,"agent":"grok-bot/executor","claims":["the test exercises the real approval path end to end (built CLI, chat-completions mock provider over HTTP, non-interactive auto-approval policy, real shell side effects)","it catches the original approval-reuse bug: same-id cases fail when the per-call approval lookup is reverted, missing-id cases fail when the adapter id fallback is reverted, all four fail with both reverted","it is not a unit test hidden in an integration file: it spawns the compiled CLI in a child process and asserts only filesystem side effects, the child's stderr, and the HTTP requests the provider mock received","no state leaks into the real ~/.term2, ~/.local/state/term2-nodejs or project .term2"],"methods":["integration-test","fault-injection","before-after-comparison","temporary-script","typecheck","lint","repeat-run"],"verification":{"minutes":17,"tokens":null,"independent":true,"verifier":"Sol","verdict":"accept","rework_rounds":0},"uncertain":["only the non-interactive --auto-approve answer path is covered; the TUI, control-socket, ACP and gateway approval paths are not exercised by this test","the missing-id cases go through the openai-compatible chat-completions adapter only; the grok chat path shares the adapter but is not run","the test depends on the shell safety classifier keeping `echo > file` GREEN and `rm file` RED; a classifier change could make it fail for unrelated reasons (it would fail loudly, not pass vacuously)","with a repeated call id, non-interactive stderr omits the `[tool] shell: ` line for the later call (the id-less runs show it); cause: SessionToolTracker.dedupeToolStarted (source/services/session/session-tool-tracker.ts:254-270) drops tool_started events whose callId was already seen, so it is display only and does not affect approval"],"human_interventions":0,"notes":"Human help: none; the conductor/Duc only gave the brief. Test-only change, no production code touched. Red/green evidence in /workspace/redproof/t13/ on the executor box. verified_at is the executor's end time. Sol verified independently and accepted with no blockers (reproduced red on pre-fix a4cc28ec, ran non-vacuity controls, gates 10/10); minutes = 8 executor + 9 Sol."} {"id":"2026-10-08-repeated-id-tool-line","type":"bugfix","risk":"high","started_at":"2026-10-08T21:33:46+07:00","verified_at":"2026-10-08T23:09:00+07:00","commit":"bb58a491","files_changed":6,"lines_changed":383,"agent":"coder","claims":["a tool call that executes is shown (non-interactive `[tool]` line, saved tool_started and command_message) even when the provider reused an earlier call's id","the duplicate the dedupe exists for (tool_started re-emitted by the approval/continuation plan for the same execution) stays suppressed: one execution, one `[tool]` line","a call that reuses the id of a refused call runs and is saved as run (isApprovalRejection false, success true); the refused call is still saved and reported as refused; approval decisions are unchanged","nothing else changes: model requests, approvals and files written are identical across 7 scenarios plus the reuse-after-refusal scenario; saved session events are identical for non-repeated runs and gain only the missing tool_started and command_message for repeated-id runs","the TUI renders the repeated-id row within one turn, and renders a later turn's call that reuses an earlier call id (one row; the row now has its own id per execution, the provider call id is unchanged); same-turn repeats, distinct-id controls and approval prompts render the same as before that change","the committed tests catch the bug: on main 3 of the original 6 integration tests fail (the 3 repeated-id cases; the 2 missing-id cases and the one-line control pass on main by design), and the reuse-after-refusal test fails on main and on 52e7809b; the TUI e2e test fails with only the row-id change reverted and passes on main"],"methods":["trace-inspection","integration-test","e2e-test","fault-injection","before-after-comparison","temporary-script","run-app","non-interactive-run","pty-run","typecheck","lint","repeat-run"],"verification":{"minutes":48,"tokens":null,"independent":true,"verifier":"Sol","verdict":"accept","rework_rounds":1},"uncertain":["TUI: when the repeated-id execution happens in the same TUI turn as the earlier call, its in-flight Running row never appears (0/4 runs; the distinct-id control showed it 4/4); the row appears once completed. Likely the TUI's per-turn seen-id set (conversation-event-handler.ts:316); not changed","two calls with the same id inside one model response: the second's tool_started is printed and saved, but its command_message is not (no result is recorded between them); not covered by a committed test","the two duplicates the original dedupe commit (c0679bcf) targeted, an SDK replay of function_call on continuation and the same function_call twice in one stream, were not observed with the chat-completions provider; Responses/codex providers were not run. A replay arriving after the call's result would now print a second line","subagent_tool_started dedupe (agentId:callId) is unchanged and may hide repeated-id subagent calls the same way","cross-turn repeats are exercised only in the TUI (e2e test and pty probes); the non-interactive runs are single-turn","after --resume, a later turn's repeated-id row is not rendered; main, the pre-row-id head and the final head render the same resumed screen. Both executions' saved command_message share the id call_0-0; fixing it would touch persisted ids or the restore path, so it is out of scope","pre-existing, same on main: in the reuse-after-refusal run the second call's saved tool_result output shows the first call's rejection text (the logger looks the result up by callId in the ledger, session-stream-processor.ts ~273)","the TUI e2e test matches the completed row in the pty's stripped output stream, not an emulated screen; exactly one row per execution was checked with the pyte screen probe (3/3), not by the committed test","saved-event equality is after normalizing uuids, timestamps and the shell's `Runtime: Nms`"],"human_interventions":3,"notes":"Three human interventions, all tagged approval and none counted (not rescue): (0) Duc's yes to queue the task at 21:33; (1) the first attempt stopped at 21:39:15 because the fix changes persisted session events (the brief's stop condition); Duc chose option A+ and the go-ahead arrived 21:44. (2) Rework round 1 after Sol rejected 52e7809b: B1, a later-turn repeated-id call ran with no TUI row (main showed it; regression from 15e0b309); B2, a call reusing a refused call's id was saved with isApprovalRejection true although it ran; nits: the turn-workflow.ts change was unobservable (dropped), uncertain[1] was wrong, the TUI claim was overstated, and the red count is 3/6 on main. Rework started 22:27:21; B1 was put in scope at 22:27 by Duc as a row-id-only TUI change. No other human help. Fix commits: 15e0b309 (forget-on-result), 7e98b7fd (B2 rejection marker cleared for a new execution; turn-workflow change dropped), 2c9b7cc7 (B1 per-execution TUI row id + e2e test), bb58a491 (e2e test quits the TUI before cleanup). files_changed/lines_changed are the combined diff vs origin/main d93142ea, excluding this file. Evidence in /workspace/redproof/t14/ (rework/ and b1/) on the executor box. Sol accepted rework round 1 at 3082eba9 with no blockers (review 22:47-23:09 ICT); verification minutes 48 = Sol's 26 in round 1 + 22 in round 2."} +{"id":"2026-10-08-bdv-approval-call-binding","type":"infra","risk":"medium","started_at":"2026-10-08T22:38:00+07:00","verified_at":"2026-10-08T22:46:02+07:00","commit":"0425b5a1","files_changed":4,"lines_changed":139,"agent":"codex/gpt-6","claims":["a frozen approval-call-binding contract maps to a real CLI integration probe with objective acceptance counts","the BDV runner records the commit, command, elapsed time, fixed test counts, output and SHA-256 digests, and passes only for the expected result","existing tests and product behavior remain unchanged"],"methods":["integration-test","before-after-comparison","lint","static-guard","evidence-ledger"],"verification":{"minutes":15,"tokens":null,"independent":false,"verifier":null,"verdict":"not-run","rework_rounds":1},"uncertain":["the attached bdv-critique.md was not available at /mnt/data/bdv-critique.md; the pilot followed the critique risks summarized in the conversation preview","the contract has not yet had a separate reviewer challenge its claim or criteria; the verifier itself only evaluates the fixed counts and probe exit status","the pilot covers one approval path and four call-id cases, not every interactive, ACP, gateway or provider path","one baseline and one pilot run establish repeatability, not a general verification cost or defect-detection comparison","the full integration tier baseline had five unrelated failures: three gateway runtime binding cases, one gateway Unix socket startup case, and one ACP session listing case"],"human_interventions":0,"notes":"Focused baseline at parent d93142ea: 1 integration file, 4 passed, 30 skipped; Vitest 2.60s. Post-commit BDV run at 0425b5a1: 1 file, 4 passed, 30 skipped; Vitest 2.48s, runner duration 2.95s. pnpm test passed 7 guards/28 cases; pnpm check:no-unit-tests passed; pnpm lint passed with 31 existing warnings."} diff --git a/package.json b/package.json index 1c78d2cf..8e8f6175 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "test:integration": "cross-env NODE_ENV=test vitest run --reporter=minimal --config vitest.integration.config.ts", "test:codex-network": "cross-env NODE_ENV=test vitest run --reporter=minimal --config vitest.e2e.config.ts scripts/fake-codex-server.e2e.test.ts", "test:provider-black-box": "cross-env NODE_ENV=test node scripts/run-provider-black-box.mjs", + "bdv": "node scripts/bdv/run.mjs", "provider:record": "tsx scripts/provider-record.ts", "provider:fixture:prepare": "tsx scripts/provider-fixture-prepare.ts", "provider:fixture:scan": "cross-env NODE_ENV=test vitest run --reporter=minimal scripts/provider-black-box/fixture-security.test.ts", diff --git a/scripts/bdv/run.mjs b/scripts/bdv/run.mjs new file mode 100644 index 00000000..5769dd59 --- /dev/null +++ b/scripts/bdv/run.mjs @@ -0,0 +1,220 @@ +#!/usr/bin/env node +import { createHash } from 'node:crypto'; +import { appendFile, mkdir, mkdtemp, readFile, rm } from 'node:fs/promises'; +import { spawnSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const claimFile = path.join(root, 'docs/experiments/bdv-pilot/claims.json'); +const evidencePath = 'docs/experiments/bdv-pilot/evidence.jsonl'; +const evidenceFile = path.join(root, evidencePath); +const probePath = 'source/cli.integration.test.ts'; +const runnerPath = 'scripts/bdv/run.mjs'; +const timeoutMs = 120_000; +const probes = { + 'approval-reuse-integration': [ + 'exec', + 'vitest', + 'run', + '--reporter=json', + '--config', + 'vitest.integration.config.ts', + probePath, + '-t', + 'an approved shell call does not approve a later call', + ], +}; + +export function digest(value) { + return createHash('sha256').update(value).digest('hex'); +} + +export function changedPaths(statusOutput) { + return statusOutput + .split('\n') + .filter(Boolean) + .map((line) => line.slice(3)) + .filter((file) => file !== evidencePath); +} + +export function runProcess(executable, args, { timeout = timeoutMs, env = process.env } = {}) { + return spawnSync(executable, args, { + cwd: root, + encoding: 'utf8', + env: { ...env, CI: '1', FORCE_COLOR: '0', NODE_ENV: 'test' }, + maxBuffer: 16 * 1024 * 1024, + timeout, + killSignal: 'SIGTERM', + }); +} + +export function evaluateRun({ exitCode, report, expectedCases, inputsStable }) { + const assertions = (report?.testResults ?? []).flatMap((suite) => suite.assertionResults ?? []); + const scenarioResults = expectedCases.map((fullName) => { + const matches = assertions.filter((result) => result.fullName === fullName); + return { + fullName, + status: matches.length === 1 ? matches[0].status : matches.length === 0 ? 'missing' : 'duplicate', + }; + }); + const accepted = + exitCode === 0 && + inputsStable && + expectedCases.length > 0 && + new Set(expectedCases).size === expectedCases.length && + scenarioResults.length === expectedCases.length && + scenarioResults.every((result) => result.status === 'passed'); + return { accepted, scenarioResults }; +} + +function git(args) { + return spawnSync('git', args, { cwd: root, encoding: 'utf8' }); +} + +async function captureInputs() { + const [claimBytes, probeBytes, runnerBytes, status] = await Promise.all([ + readFile(claimFile), + readFile(path.join(root, probePath)), + readFile(path.join(root, runnerPath)), + Promise.resolve(git(['status', '--porcelain=v1', '--untracked-files=all'])), + ]); + const head = git(['rev-parse', 'HEAD']); + return { + commit: head.status === 0 ? head.stdout.trim() : null, + claimDigest: digest(claimBytes), + probeDigest: digest(probeBytes), + runnerDigest: digest(runnerBytes), + dirtyPaths: status.status === 0 ? changedPaths(status.stdout) : [''], + }; +} + +function sameInputs(before, after) { + return ( + before.commit !== null && + before.commit === after.commit && + before.claimDigest === after.claimDigest && + before.probeDigest === after.probeDigest && + before.runnerDigest === after.runnerDigest && + before.dirtyPaths.length === 0 && + after.dirtyPaths.length === 0 + ); +} + +function normalizeOutput(value, tempDirectory) { + return value + .replace(/\u001b\[[0-9;]*m/g, '') + .replaceAll(root, '') + .replaceAll(tempDirectory, ''); +} + +async function main() { + const claimId = process.argv[2]; + const claimBytes = await readFile(claimFile); + const claims = JSON.parse(claimBytes.toString('utf8')).claims; + const claim = claims.find((item) => item.id === claimId); + if (!claim || !probes[claim.probe]) { + console.error(`Usage: pnpm bdv \nAvailable claims: ${claims.map((item) => item.id).join(', ')}`); + process.exitCode = 2; + return; + } + + const startedAt = new Date(); + const started = performance.now(); + const before = await captureInputs(); + const approvedProbe = before.probeDigest === claim.approvedProbeDigest; + const preflightPassed = + before.dirtyPaths.length === 0 && + before.claimDigest === digest(claimBytes) && + approvedProbe && + before.commit !== null; + let result = { status: 1, signal: null, error: null, stdout: '', stderr: '' }; + let report = null; + let command = ['pnpm', ...probes[claim.probe], '--outputFile', '/vitest-report.json']; + let tempDirectory = ''; + + if (preflightPassed) { + tempDirectory = await mkdtemp(path.join(tmpdir(), 'term2-bdv-')); + const reportFile = path.join(tempDirectory, 'vitest-report.json'); + command = ['pnpm', ...probes[claim.probe], '--outputFile', reportFile]; + try { + result = runProcess('pnpm', [...probes[claim.probe], '--outputFile', reportFile], { timeout: timeoutMs }); + try { + const reportBytes = await readFile(reportFile); + report = JSON.parse(reportBytes.toString('utf8')); + } catch { + report = null; + } + } finally { + await rm(tempDirectory, { recursive: true, force: true }); + } + } + + const after = await captureInputs(); + const inputsStable = preflightPassed && sameInputs(before, after); + const exitCode = result.status ?? 1; + const evaluated = evaluateRun({ + exitCode, + report, + expectedCases: claim.acceptance.expectedCases, + inputsStable, + }); + const durationMs = Math.round(performance.now() - started); + const structuredResult = { + totalTests: report?.numTotalTests ?? null, + passedTests: report?.numPassedTests ?? null, + failedTests: report?.numFailedTests ?? null, + scenarioResults: evaluated.scenarioResults, + }; + const output = normalizeOutput(`${result.stdout ?? ''}${result.stderr ?? ''}`, tempDirectory); + const evidence = { + schemaVersion: 2, + claimId, + claimDigest: before.claimDigest, + approvedProbeDigest: claim.approvedProbeDigest, + probeDigest: before.probeDigest, + runnerDigest: before.runnerDigest, + commit: before.commit, + commitAfter: after.commit, + claimDigestAfter: after.claimDigest, + probeDigestAfter: after.probeDigest, + runnerDigestAfter: after.runnerDigest, + inputsStable, + dirtyPathsBefore: before.dirtyPaths, + dirtyPathsAfter: after.dirtyPaths, + startedAt: startedAt.toISOString(), + durationMs, + timeoutMs, + timedOut: result.error?.code === 'ETIMEDOUT', + command: command.map((part) => (part.startsWith(tempDirectory) ? '/vitest-report.json' : part)), + exitCode, + ...structuredResult, + resultDigest: digest(JSON.stringify(structuredResult)), + verdict: evaluated.accepted ? 'pass' : 'fail', + outputDigest: digest(output), + output, + error: result.error ? { code: result.error.code, message: result.error.message } : null, + }; + + await mkdir(path.dirname(evidenceFile), { recursive: true }); + await appendFile(evidenceFile, `${JSON.stringify(evidence)}\n`); + process.stdout.write(output); + console.log(`BDV ${claimId}: ${evidence.verdict}; evidence appended to ${evidencePath}`); + if (!preflightPassed) { + console.error( + `BDV refused to run: ${ + !approvedProbe + ? 'probe digest differs from the frozen contract' + : 'working tree is dirty or Git state is unavailable' + }`, + ); + } else if (!inputsStable) { + console.error('BDV rejected the run because executable inputs changed during the probe'); + } + process.exitCode = evaluated.accepted ? 0 : 1; +} + +if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) { + await main(); +} diff --git a/source/cli.integration.test.ts b/source/cli.integration.test.ts index 244dd017..69bae14a 100644 --- a/source/cli.integration.test.ts +++ b/source/cli.integration.test.ts @@ -11,6 +11,11 @@ import { createTestChildEnv } from './test-helpers/terminal-e2e.js'; const require = createRequire(import.meta.url); +function expectedToolLine(command: string): string { + const preview = command.length > 80 ? `${command.slice(0, 80)}…` : command; + return `[tool] shell: ${preview}`; +} + // dist/ is a gitignored build artifact, so a fresh worktree has no compiled // CLI and these tests would fail until someone runs `pnpm build`. Instead of // spawning tsx per test (which pays a ~2s compile per process), compile the @@ -1327,10 +1332,7 @@ function expectSecondCallRefused(run: Awaited `[tool] shell: ${command}`)); + expect(run.toolLines, report).toEqual(run.commands.map(expectedToolLine)); // The saved session log (what resume replays) records both executions and both outputs, once each. expect(run.savedToolStarts).toEqual(run.commands); expect(run.savedCommandOutputs).toEqual(run.commands); @@ -1415,7 +1417,7 @@ it('one approved shell execution prints exactly one [tool] line', { timeout: 90_ expect(run.status, report).toBe(0); expect(run.files, report).toEqual({ 'one.txt': 'one\n' }); expect(run.stderr.match(/\[approval required\] shell/g), report).toHaveLength(1); - expect(run.toolLines, report).toEqual([`[tool] shell: ${run.commands[0]}`]); + expect(run.toolLines, report).toEqual([expectedToolLine(run.commands[0])]); expect(run.savedToolStarts).toEqual(run.commands); expect(run.savedCommandOutputs).toEqual(run.commands); }); @@ -1437,7 +1439,7 @@ it('a call that reuses a refused call id runs and is saved as run, not as refuse const report = `exit ${run.status}\nstderr:\n${run.stderr}`; expect(run.status, report).toBe(0); expect(run.files, report).toEqual({ 'two.txt': 'two\n', 'victim.txt': 'must survive\n' }); - expect(run.toolLines, report).toEqual([`[tool] shell: ${refusedCommand}`, `[tool] shell: ${ranCommand}`]); + expect(run.toolLines, report).toEqual([expectedToolLine(refusedCommand), expectedToolLine(ranCommand)]); // Only the first call was refused. expect(run.stderr.match(/Approval Rejected:/g), report).toHaveLength(1); expect(run.stderr, report).toContain(`cannot be executed automatically: ${refusedCommand}`);