From 5ee7a1642eafc5fd7d9944526c137ea295fb0823 Mon Sep 17 00:00:00 2001 From: qduc Date: Thu, 8 Oct 2026 22:45:50 +0700 Subject: [PATCH 1/7] feat(bdv): add approval claim evidence pilot --- docs/experiments/bdv-pilot/README.md | 32 ++++++++++ docs/experiments/bdv-pilot/claims.json | 22 +++++++ package.json | 1 + scripts/bdv/run.mjs | 84 ++++++++++++++++++++++++++ 4 files changed, 139 insertions(+) create mode 100644 docs/experiments/bdv-pilot/README.md create mode 100644 docs/experiments/bdv-pilot/claims.json create mode 100644 scripts/bdv/run.mjs diff --git a/docs/experiments/bdv-pilot/README.md b/docs/experiments/bdv-pilot/README.md new file mode 100644 index 00000000..e854acdd --- /dev/null +++ b/docs/experiments/bdv-pilot/README.md @@ -0,0 +1,32 @@ +# BDV pilot: approval call binding + +This pilot links a frozen behavior claim to a retained integration probe and a machine-readable evidence ledger. It does not change product behavior or replace existing tests. + +## Run + +```sh +pnpm bdv approval-call-binding +``` + +The runner executes the selected Vitest integration case, compares the reported test and file counts with the fixed acceptance counts in [`claims.json`](claims.json), and appends the command, exit code, elapsed time, commit, SHA-256 digests of the claim/probe/runner, result counts, output digest, and output to [`evidence.jsonl`](evidence.jsonl). A later reviewer can verify the contract and artifacts, inspect the probe assertions, compare their hashes, and rerun the command. + +## Independence and limits + +Acceptance is a fixed contract that the runner does not generate from the observed output. It expresses the approval rule as externally observable outcomes: only the allowed file is created, the denied target survives, each tool call receives its own decision, and the provider sees paired calls and results. The probe crosses the built CLI, a loopback HTTP provider fixture, and isolated filesystem effects. Hashes expose changes to the contract or probe between recorded runs. + +This is one claim and one probe, not a general probe scheduler. The claim contract and its probe are maintained in the same repository, so hashes establish provenance and change detection, not independent authorship. An independent reviewer has not yet challenged this pilot contract. It does not yet enforce independent contract approval, compare costs across broad test suites, or show that passing the selected scenarios proves all approval paths safe. + +## Baseline and pilot result + +Baseline at `d93142ea` (before the runner), using the same probe directly: + +```text +pnpm exec vitest run --reporter=minimal --config vitest.integration.config.ts source/cli.integration.test.ts -t 'an approved shell call does not approve a later call' +Test Files 1 passed (1) +Tests 4 passed | 30 skipped (34) +Duration 2.60s +``` + +The baseline establishes repeatability and elapsed test time only; it is not a full-project defect-detection or cost comparison. The pilot evidence record is appended when `pnpm bdv approval-call-binding` runs. + +The first runner execution result and elapsed time are captured in the evidence ledger. Compare them with the baseline as a one-run observation only; this pilot does not establish a performance trend. diff --git a/docs/experiments/bdv-pilot/claims.json b/docs/experiments/bdv-pilot/claims.json new file mode 100644 index 00000000..fd0b5d71 --- /dev/null +++ b/docs/experiments/bdv-pilot/claims.json @@ -0,0 +1,22 @@ +{ + "schemaVersion": 1, + "claims": [ + { + "id": "approval-call-binding", + "statement": "An approval for one shell tool call cannot authorize a later call that reuses the same provider call id, and adapter-generated ids for missing or empty provider ids remain distinct.", + "authority": "Frozen pilot contract derived from the existing user-facing approval policy and the expected safe side effects; this runner cannot alter the expected outcomes.", + "probe": "approval-reuse-integration", + "acceptance": { + "passedFiles": 1, + "passedTests": 4, + "mustAssert": [ + "the first GREEN unsandboxed command runs and writes exactly 'approved\\n'", + "the subsequent RED rm command is refused and its victim file remains exactly 'must survive\\n'", + "the CLI reports two distinct approval decisions and the RED refusal reason", + "all four provider-id, repeated-id, omitted-id, and empty-id scenarios pass", + "the provider receives both calls and paired tool results" + ] + } + } + ] +} diff --git a/package.json b/package.json index 1c78d2cf..8e8f6175 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "test:integration": "cross-env NODE_ENV=test vitest run --reporter=minimal --config vitest.integration.config.ts", "test:codex-network": "cross-env NODE_ENV=test vitest run --reporter=minimal --config vitest.e2e.config.ts scripts/fake-codex-server.e2e.test.ts", "test:provider-black-box": "cross-env NODE_ENV=test node scripts/run-provider-black-box.mjs", + "bdv": "node scripts/bdv/run.mjs", "provider:record": "tsx scripts/provider-record.ts", "provider:fixture:prepare": "tsx scripts/provider-fixture-prepare.ts", "provider:fixture:scan": "cross-env NODE_ENV=test vitest run --reporter=minimal scripts/provider-black-box/fixture-security.test.ts", diff --git a/scripts/bdv/run.mjs b/scripts/bdv/run.mjs new file mode 100644 index 00000000..101a3c9b --- /dev/null +++ b/scripts/bdv/run.mjs @@ -0,0 +1,84 @@ +#!/usr/bin/env node +import { createHash } from 'node:crypto'; +import { appendFile, mkdir, readFile } from 'node:fs/promises'; +import { spawnSync } from 'node:child_process'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const claimFile = path.join(root, 'docs/experiments/bdv-pilot/claims.json'); +const evidenceFile = path.join(root, 'docs/experiments/bdv-pilot/evidence.jsonl'); +const probeArgs = { + 'approval-reuse-integration': [ + 'exec', + 'vitest', + 'run', + '--reporter=minimal', + '--config', + 'vitest.integration.config.ts', + 'source/cli.integration.test.ts', + '-t', + 'an approved shell call does not approve a later call', + ], +}; + +function digest(value) { + return createHash('sha256').update(value).digest('hex'); +} + +function parseCount(output, label) { + const match = output.match(new RegExp(`${label}\\s+(\\d+) passed`)); + return match ? Number(match[1]) : null; +} + +const claimId = process.argv[2]; +const claims = JSON.parse(await readFile(claimFile, 'utf8')).claims; +const claim = claims.find((item) => item.id === claimId); +if (!claim || !probeArgs[claim.probe]) { + console.error(`Usage: pnpm bdv \nAvailable claims: ${claims.map((item) => item.id).join(', ')}`); + process.exit(2); +} + +const startedAt = new Date(); +const started = performance.now(); +const result = spawnSync('pnpm', probeArgs[claim.probe], { + cwd: root, + encoding: 'utf8', + env: { ...process.env, CI: '1', FORCE_COLOR: '0', NODE_ENV: 'test' }, + maxBuffer: 16 * 1024 * 1024, +}); +const durationMs = Math.round(performance.now() - started); +const output = `${result.stdout ?? ''}${result.stderr ?? ''}` + .replace(/\u001b\[[0-9;]*m/g, '') + .replaceAll(root, ''); +const passedFiles = parseCount(output, 'Test Files'); +const passedTests = parseCount(output, 'Tests'); +const accepted = + result.status === 0 && passedFiles === claim.acceptance.passedFiles && passedTests === claim.acceptance.passedTests; + +const git = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }); +const probePath = path.join(root, 'source/cli.integration.test.ts'); +const runnerPath = fileURLToPath(import.meta.url); +const evidence = { + schemaVersion: 1, + claimId, + claimDigest: digest(await readFile(claimFile)), + probeDigest: digest(await readFile(probePath)), + runnerDigest: digest(await readFile(runnerPath)), + commit: git.status === 0 ? git.stdout.trim() : null, + startedAt: startedAt.toISOString(), + durationMs, + command: ['pnpm', ...probeArgs[claim.probe]], + exitCode: result.status ?? 1, + passedFiles, + passedTests, + verdict: accepted ? 'pass' : 'fail', + outputDigest: digest(output), + output, +}; + +await mkdir(path.dirname(evidenceFile), { recursive: true }); +await appendFile(evidenceFile, `${JSON.stringify(evidence)}\n`); +process.stdout.write(output); +console.log(`BDV ${claimId}: ${evidence.verdict}; evidence appended to docs/experiments/bdv-pilot/evidence.jsonl`); +process.exitCode = accepted ? 0 : 1; From 2a00a72dd5bc9186c432e06ab25bc38f9ca2d296 Mon Sep 17 00:00:00 2001 From: qduc Date: Thu, 8 Oct 2026 22:48:03 +0700 Subject: [PATCH 2/7] docs(bdv): record pilot verification evidence --- docs/experiments/bdv-pilot/evidence.jsonl | 1 + docs/experiments/evidence-driven-development/tasks.jsonl | 1 + 2 files changed, 2 insertions(+) create mode 100644 docs/experiments/bdv-pilot/evidence.jsonl diff --git a/docs/experiments/bdv-pilot/evidence.jsonl b/docs/experiments/bdv-pilot/evidence.jsonl new file mode 100644 index 00000000..8abe085e --- /dev/null +++ b/docs/experiments/bdv-pilot/evidence.jsonl @@ -0,0 +1 @@ +{"schemaVersion":1,"claimId":"approval-call-binding","claimDigest":"aa52df0624b9888e789272e09bf21d8126663b4802664671afcfffb3788d5d07","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigest":"c8ed6477c63fc2507a7979a6c92db43c89d19c8f48003465626d67808d2e0be6","commit":"0425b5a12e6b4429f3621d16a6f3c38205e0dc5e","startedAt":"2026-10-08T15:45:59.677Z","durationMs":2900,"command":["pnpm","exec","vitest","run","--reporter=minimal","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call"],"exitCode":0,"passedFiles":1,"passedTests":4,"verdict":"pass","outputDigest":"b89ec2955b9af86381110d1be4d22669b178c74c2cc45f149e87728f4a71e16d","output":"\n RUN v4.1.9 \n\n\n Test Files 1 passed (1)\n Tests 4 passed | 30 skipped (34)\n Start at 22:45:59\n Duration 2.59s (transform 31ms, setup 12ms, import 36ms, tests 2.48s, environment 0ms)\n\n"} diff --git a/docs/experiments/evidence-driven-development/tasks.jsonl b/docs/experiments/evidence-driven-development/tasks.jsonl index aac458a9..f40f51ad 100644 --- a/docs/experiments/evidence-driven-development/tasks.jsonl +++ b/docs/experiments/evidence-driven-development/tasks.jsonl @@ -1,2 +1,3 @@ {"id":"2026-10-08-approval-reuse-guard","type":"infra","risk":"high","started_at":"2026-10-08T20:52:58+07:00","verified_at":"2026-10-08T21:00:53+07:00","commit":"6ccef478","files_changed":1,"lines_changed":166,"agent":"grok-bot/executor","claims":["the test exercises the real approval path end to end (built CLI, chat-completions mock provider over HTTP, non-interactive auto-approval policy, real shell side effects)","it catches the original approval-reuse bug: same-id cases fail when the per-call approval lookup is reverted, missing-id cases fail when the adapter id fallback is reverted, all four fail with both reverted","it is not a unit test hidden in an integration file: it spawns the compiled CLI in a child process and asserts only filesystem side effects, the child's stderr, and the HTTP requests the provider mock received","no state leaks into the real ~/.term2, ~/.local/state/term2-nodejs or project .term2"],"methods":["integration-test","fault-injection","before-after-comparison","temporary-script","typecheck","lint","repeat-run"],"verification":{"minutes":17,"tokens":null,"independent":true,"verifier":"Sol","verdict":"accept","rework_rounds":0},"uncertain":["only the non-interactive --auto-approve answer path is covered; the TUI, control-socket, ACP and gateway approval paths are not exercised by this test","the missing-id cases go through the openai-compatible chat-completions adapter only; the grok chat path shares the adapter but is not run","the test depends on the shell safety classifier keeping `echo > file` GREEN and `rm file` RED; a classifier change could make it fail for unrelated reasons (it would fail loudly, not pass vacuously)","with a repeated call id, non-interactive stderr omits the `[tool] shell: ` line for the later call (the id-less runs show it); cause: SessionToolTracker.dedupeToolStarted (source/services/session/session-tool-tracker.ts:254-270) drops tool_started events whose callId was already seen, so it is display only and does not affect approval"],"human_interventions":0,"notes":"Human help: none; the conductor/Duc only gave the brief. Test-only change, no production code touched. Red/green evidence in /workspace/redproof/t13/ on the executor box. verified_at is the executor's end time. Sol verified independently and accepted with no blockers (reproduced red on pre-fix a4cc28ec, ran non-vacuity controls, gates 10/10); minutes = 8 executor + 9 Sol."} {"id":"2026-10-08-repeated-id-tool-line","type":"bugfix","risk":"high","started_at":"2026-10-08T21:33:46+07:00","verified_at":"2026-10-08T23:09:00+07:00","commit":"bb58a491","files_changed":6,"lines_changed":383,"agent":"coder","claims":["a tool call that executes is shown (non-interactive `[tool]` line, saved tool_started and command_message) even when the provider reused an earlier call's id","the duplicate the dedupe exists for (tool_started re-emitted by the approval/continuation plan for the same execution) stays suppressed: one execution, one `[tool]` line","a call that reuses the id of a refused call runs and is saved as run (isApprovalRejection false, success true); the refused call is still saved and reported as refused; approval decisions are unchanged","nothing else changes: model requests, approvals and files written are identical across 7 scenarios plus the reuse-after-refusal scenario; saved session events are identical for non-repeated runs and gain only the missing tool_started and command_message for repeated-id runs","the TUI renders the repeated-id row within one turn, and renders a later turn's call that reuses an earlier call id (one row; the row now has its own id per execution, the provider call id is unchanged); same-turn repeats, distinct-id controls and approval prompts render the same as before that change","the committed tests catch the bug: on main 3 of the original 6 integration tests fail (the 3 repeated-id cases; the 2 missing-id cases and the one-line control pass on main by design), and the reuse-after-refusal test fails on main and on 52e7809b; the TUI e2e test fails with only the row-id change reverted and passes on main"],"methods":["trace-inspection","integration-test","e2e-test","fault-injection","before-after-comparison","temporary-script","run-app","non-interactive-run","pty-run","typecheck","lint","repeat-run"],"verification":{"minutes":48,"tokens":null,"independent":true,"verifier":"Sol","verdict":"accept","rework_rounds":1},"uncertain":["TUI: when the repeated-id execution happens in the same TUI turn as the earlier call, its in-flight Running row never appears (0/4 runs; the distinct-id control showed it 4/4); the row appears once completed. Likely the TUI's per-turn seen-id set (conversation-event-handler.ts:316); not changed","two calls with the same id inside one model response: the second's tool_started is printed and saved, but its command_message is not (no result is recorded between them); not covered by a committed test","the two duplicates the original dedupe commit (c0679bcf) targeted, an SDK replay of function_call on continuation and the same function_call twice in one stream, were not observed with the chat-completions provider; Responses/codex providers were not run. A replay arriving after the call's result would now print a second line","subagent_tool_started dedupe (agentId:callId) is unchanged and may hide repeated-id subagent calls the same way","cross-turn repeats are exercised only in the TUI (e2e test and pty probes); the non-interactive runs are single-turn","after --resume, a later turn's repeated-id row is not rendered; main, the pre-row-id head and the final head render the same resumed screen. Both executions' saved command_message share the id call_0-0; fixing it would touch persisted ids or the restore path, so it is out of scope","pre-existing, same on main: in the reuse-after-refusal run the second call's saved tool_result output shows the first call's rejection text (the logger looks the result up by callId in the ledger, session-stream-processor.ts ~273)","the TUI e2e test matches the completed row in the pty's stripped output stream, not an emulated screen; exactly one row per execution was checked with the pyte screen probe (3/3), not by the committed test","saved-event equality is after normalizing uuids, timestamps and the shell's `Runtime: Nms`"],"human_interventions":3,"notes":"Three human interventions, all tagged approval and none counted (not rescue): (0) Duc's yes to queue the task at 21:33; (1) the first attempt stopped at 21:39:15 because the fix changes persisted session events (the brief's stop condition); Duc chose option A+ and the go-ahead arrived 21:44. (2) Rework round 1 after Sol rejected 52e7809b: B1, a later-turn repeated-id call ran with no TUI row (main showed it; regression from 15e0b309); B2, a call reusing a refused call's id was saved with isApprovalRejection true although it ran; nits: the turn-workflow.ts change was unobservable (dropped), uncertain[1] was wrong, the TUI claim was overstated, and the red count is 3/6 on main. Rework started 22:27:21; B1 was put in scope at 22:27 by Duc as a row-id-only TUI change. No other human help. Fix commits: 15e0b309 (forget-on-result), 7e98b7fd (B2 rejection marker cleared for a new execution; turn-workflow change dropped), 2c9b7cc7 (B1 per-execution TUI row id + e2e test), bb58a491 (e2e test quits the TUI before cleanup). files_changed/lines_changed are the combined diff vs origin/main d93142ea, excluding this file. Evidence in /workspace/redproof/t14/ (rework/ and b1/) on the executor box. Sol accepted rework round 1 at 3082eba9 with no blockers (review 22:47-23:09 ICT); verification minutes 48 = Sol's 26 in round 1 + 22 in round 2."} +{"id":"2026-10-08-bdv-approval-call-binding","type":"infra","risk":"medium","started_at":"2026-10-08T22:38:00+07:00","verified_at":"2026-10-08T22:46:02+07:00","commit":"0425b5a1","files_changed":4,"lines_changed":139,"agent":"codex/gpt-6","claims":["a frozen approval-call-binding contract maps to a real CLI integration probe with objective acceptance counts","the BDV runner records the commit, command, elapsed time, fixed test counts, output and SHA-256 digests, and passes only for the expected result","existing tests and product behavior remain unchanged"],"methods":["integration-test","before-after-comparison","lint","static-guard","evidence-ledger"],"verification":{"minutes":15,"tokens":null,"independent":false,"verifier":null,"verdict":"not-run","rework_rounds":1},"uncertain":["the attached bdv-critique.md was not available at /mnt/data/bdv-critique.md; the pilot followed the critique risks summarized in the conversation preview","the contract has not yet had a separate reviewer challenge its claim or criteria; the verifier itself only evaluates the fixed counts and probe exit status","the pilot covers one approval path and four call-id cases, not every interactive, ACP, gateway or provider path","one baseline and one pilot run establish repeatability, not a general verification cost or defect-detection comparison","the full integration tier baseline had five unrelated failures: three gateway runtime binding cases, one gateway Unix socket startup case, and one ACP session listing case"],"human_interventions":0,"notes":"Focused baseline at parent d93142ea: 1 integration file, 4 passed, 30 skipped; Vitest 2.60s. Post-commit BDV run at 0425b5a1: 1 file, 4 passed, 30 skipped; Vitest 2.48s, runner duration 2.95s. pnpm test passed 7 guards/28 cases; pnpm check:no-unit-tests passed; pnpm lint passed with 31 existing warnings."} From abc5a6543f6a496554c645a7c512a70c9ba52cb9 Mon Sep 17 00:00:00 2001 From: qduc Date: Thu, 8 Oct 2026 22:59:47 +0700 Subject: [PATCH 3/7] fix(bdv): bind verdicts to frozen probe evidence --- docs/experiments/bdv-pilot/README.md | 6 +- docs/experiments/bdv-pilot/claims.json | 9 +- scripts/bdv/run.mjs | 250 +++++++++++++++++++------ 3 files changed, 204 insertions(+), 61 deletions(-) diff --git a/docs/experiments/bdv-pilot/README.md b/docs/experiments/bdv-pilot/README.md index e854acdd..ba65c82f 100644 --- a/docs/experiments/bdv-pilot/README.md +++ b/docs/experiments/bdv-pilot/README.md @@ -8,11 +8,13 @@ This pilot links a frozen behavior claim to a retained integration probe and a m pnpm bdv approval-call-binding ``` -The runner executes the selected Vitest integration case, compares the reported test and file counts with the fixed acceptance counts in [`claims.json`](claims.json), and appends the command, exit code, elapsed time, commit, SHA-256 digests of the claim/probe/runner, result counts, output digest, and output to [`evidence.jsonl`](evidence.jsonl). A later reviewer can verify the contract and artifacts, inspect the probe assertions, compare their hashes, and rerun the command. +The runner requires a clean checkout (the evidence ledger itself may already contain earlier runs) and checks that the probe matches the digest frozen in [`claims.json`](claims.json). It then runs Vitest's JSON reporter with a 120-second process timeout and requires each of the four exact scenario names to appear exactly once with `passed` status. Aggregate counts or earlier console text cannot produce a passing verdict. + +Before and after the probe, it records the commit and hashes the claim, probe, and runner. It rejects dirty executable inputs and any change to those hashes or `HEAD` during execution. The appended ledger entry includes before/after input hashes, the command, timeout and exit status, exact scenario results, result digest, elapsed time, output digest, and normalized output. This makes the observed result checkable against the committed contract and probe. ## Independence and limits -Acceptance is a fixed contract that the runner does not generate from the observed output. It expresses the approval rule as externally observable outcomes: only the allowed file is created, the denied target survives, each tool call receives its own decision, and the provider sees paired calls and results. The probe crosses the built CLI, a loopback HTTP provider fixture, and isolated filesystem effects. Hashes expose changes to the contract or probe between recorded runs. +Acceptance is a fixed contract that the runner does not generate from the observed output. It expresses the approval rule as externally observable outcomes: only the allowed file is created, the denied target survives, each tool call receives its own decision, and the provider sees paired calls and results. The probe crosses the built CLI, a loopback HTTP provider fixture, and isolated filesystem effects. Its approved source digest and exact scenario names bind the runner to the reviewed probe; changes require an explicit contract update. This is one claim and one probe, not a general probe scheduler. The claim contract and its probe are maintained in the same repository, so hashes establish provenance and change detection, not independent authorship. An independent reviewer has not yet challenged this pilot contract. It does not yet enforce independent contract approval, compare costs across broad test suites, or show that passing the selected scenarios proves all approval paths safe. diff --git a/docs/experiments/bdv-pilot/claims.json b/docs/experiments/bdv-pilot/claims.json index fd0b5d71..44ee7a87 100644 --- a/docs/experiments/bdv-pilot/claims.json +++ b/docs/experiments/bdv-pilot/claims.json @@ -6,9 +6,14 @@ "statement": "An approval for one shell tool call cannot authorize a later call that reuses the same provider call id, and adapter-generated ids for missing or empty provider ids remain distinct.", "authority": "Frozen pilot contract derived from the existing user-facing approval policy and the expected safe side effects; this runner cannot alter the expected outcomes.", "probe": "approval-reuse-integration", + "approvedProbeDigest": "05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328", "acceptance": { - "passedFiles": 1, - "passedTests": 4, + "expectedCases": [ + "an approved shell call does not approve a later call that reuses its id when the provider repeats call_0", + "an approved shell call does not approve a later call that reuses its id when the provider repeats its own id", + "an approved shell call does not approve a later call when the chat-completions provider omits tool call ids", + "an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids" + ], "mustAssert": [ "the first GREEN unsandboxed command runs and writes exactly 'approved\\n'", "the subsequent RED rm command is refused and its victim file remains exactly 'must survive\\n'", diff --git a/scripts/bdv/run.mjs b/scripts/bdv/run.mjs index 101a3c9b..5769dd59 100644 --- a/scripts/bdv/run.mjs +++ b/scripts/bdv/run.mjs @@ -1,84 +1,220 @@ #!/usr/bin/env node import { createHash } from 'node:crypto'; -import { appendFile, mkdir, readFile } from 'node:fs/promises'; +import { appendFile, mkdir, mkdtemp, readFile, rm } from 'node:fs/promises'; import { spawnSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; import path from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); const claimFile = path.join(root, 'docs/experiments/bdv-pilot/claims.json'); -const evidenceFile = path.join(root, 'docs/experiments/bdv-pilot/evidence.jsonl'); -const probeArgs = { +const evidencePath = 'docs/experiments/bdv-pilot/evidence.jsonl'; +const evidenceFile = path.join(root, evidencePath); +const probePath = 'source/cli.integration.test.ts'; +const runnerPath = 'scripts/bdv/run.mjs'; +const timeoutMs = 120_000; +const probes = { 'approval-reuse-integration': [ 'exec', 'vitest', 'run', - '--reporter=minimal', + '--reporter=json', '--config', 'vitest.integration.config.ts', - 'source/cli.integration.test.ts', + probePath, '-t', 'an approved shell call does not approve a later call', ], }; -function digest(value) { +export function digest(value) { return createHash('sha256').update(value).digest('hex'); } -function parseCount(output, label) { - const match = output.match(new RegExp(`${label}\\s+(\\d+) passed`)); - return match ? Number(match[1]) : null; +export function changedPaths(statusOutput) { + return statusOutput + .split('\n') + .filter(Boolean) + .map((line) => line.slice(3)) + .filter((file) => file !== evidencePath); } -const claimId = process.argv[2]; -const claims = JSON.parse(await readFile(claimFile, 'utf8')).claims; -const claim = claims.find((item) => item.id === claimId); -if (!claim || !probeArgs[claim.probe]) { - console.error(`Usage: pnpm bdv \nAvailable claims: ${claims.map((item) => item.id).join(', ')}`); - process.exit(2); +export function runProcess(executable, args, { timeout = timeoutMs, env = process.env } = {}) { + return spawnSync(executable, args, { + cwd: root, + encoding: 'utf8', + env: { ...env, CI: '1', FORCE_COLOR: '0', NODE_ENV: 'test' }, + maxBuffer: 16 * 1024 * 1024, + timeout, + killSignal: 'SIGTERM', + }); } -const startedAt = new Date(); -const started = performance.now(); -const result = spawnSync('pnpm', probeArgs[claim.probe], { - cwd: root, - encoding: 'utf8', - env: { ...process.env, CI: '1', FORCE_COLOR: '0', NODE_ENV: 'test' }, - maxBuffer: 16 * 1024 * 1024, -}); -const durationMs = Math.round(performance.now() - started); -const output = `${result.stdout ?? ''}${result.stderr ?? ''}` - .replace(/\u001b\[[0-9;]*m/g, '') - .replaceAll(root, ''); -const passedFiles = parseCount(output, 'Test Files'); -const passedTests = parseCount(output, 'Tests'); -const accepted = - result.status === 0 && passedFiles === claim.acceptance.passedFiles && passedTests === claim.acceptance.passedTests; +export function evaluateRun({ exitCode, report, expectedCases, inputsStable }) { + const assertions = (report?.testResults ?? []).flatMap((suite) => suite.assertionResults ?? []); + const scenarioResults = expectedCases.map((fullName) => { + const matches = assertions.filter((result) => result.fullName === fullName); + return { + fullName, + status: matches.length === 1 ? matches[0].status : matches.length === 0 ? 'missing' : 'duplicate', + }; + }); + const accepted = + exitCode === 0 && + inputsStable && + expectedCases.length > 0 && + new Set(expectedCases).size === expectedCases.length && + scenarioResults.length === expectedCases.length && + scenarioResults.every((result) => result.status === 'passed'); + return { accepted, scenarioResults }; +} -const git = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }); -const probePath = path.join(root, 'source/cli.integration.test.ts'); -const runnerPath = fileURLToPath(import.meta.url); -const evidence = { - schemaVersion: 1, - claimId, - claimDigest: digest(await readFile(claimFile)), - probeDigest: digest(await readFile(probePath)), - runnerDigest: digest(await readFile(runnerPath)), - commit: git.status === 0 ? git.stdout.trim() : null, - startedAt: startedAt.toISOString(), - durationMs, - command: ['pnpm', ...probeArgs[claim.probe]], - exitCode: result.status ?? 1, - passedFiles, - passedTests, - verdict: accepted ? 'pass' : 'fail', - outputDigest: digest(output), - output, -}; +function git(args) { + return spawnSync('git', args, { cwd: root, encoding: 'utf8' }); +} + +async function captureInputs() { + const [claimBytes, probeBytes, runnerBytes, status] = await Promise.all([ + readFile(claimFile), + readFile(path.join(root, probePath)), + readFile(path.join(root, runnerPath)), + Promise.resolve(git(['status', '--porcelain=v1', '--untracked-files=all'])), + ]); + const head = git(['rev-parse', 'HEAD']); + return { + commit: head.status === 0 ? head.stdout.trim() : null, + claimDigest: digest(claimBytes), + probeDigest: digest(probeBytes), + runnerDigest: digest(runnerBytes), + dirtyPaths: status.status === 0 ? changedPaths(status.stdout) : [''], + }; +} + +function sameInputs(before, after) { + return ( + before.commit !== null && + before.commit === after.commit && + before.claimDigest === after.claimDigest && + before.probeDigest === after.probeDigest && + before.runnerDigest === after.runnerDigest && + before.dirtyPaths.length === 0 && + after.dirtyPaths.length === 0 + ); +} + +function normalizeOutput(value, tempDirectory) { + return value + .replace(/\u001b\[[0-9;]*m/g, '') + .replaceAll(root, '') + .replaceAll(tempDirectory, ''); +} + +async function main() { + const claimId = process.argv[2]; + const claimBytes = await readFile(claimFile); + const claims = JSON.parse(claimBytes.toString('utf8')).claims; + const claim = claims.find((item) => item.id === claimId); + if (!claim || !probes[claim.probe]) { + console.error(`Usage: pnpm bdv \nAvailable claims: ${claims.map((item) => item.id).join(', ')}`); + process.exitCode = 2; + return; + } + + const startedAt = new Date(); + const started = performance.now(); + const before = await captureInputs(); + const approvedProbe = before.probeDigest === claim.approvedProbeDigest; + const preflightPassed = + before.dirtyPaths.length === 0 && + before.claimDigest === digest(claimBytes) && + approvedProbe && + before.commit !== null; + let result = { status: 1, signal: null, error: null, stdout: '', stderr: '' }; + let report = null; + let command = ['pnpm', ...probes[claim.probe], '--outputFile', '/vitest-report.json']; + let tempDirectory = ''; + + if (preflightPassed) { + tempDirectory = await mkdtemp(path.join(tmpdir(), 'term2-bdv-')); + const reportFile = path.join(tempDirectory, 'vitest-report.json'); + command = ['pnpm', ...probes[claim.probe], '--outputFile', reportFile]; + try { + result = runProcess('pnpm', [...probes[claim.probe], '--outputFile', reportFile], { timeout: timeoutMs }); + try { + const reportBytes = await readFile(reportFile); + report = JSON.parse(reportBytes.toString('utf8')); + } catch { + report = null; + } + } finally { + await rm(tempDirectory, { recursive: true, force: true }); + } + } -await mkdir(path.dirname(evidenceFile), { recursive: true }); -await appendFile(evidenceFile, `${JSON.stringify(evidence)}\n`); -process.stdout.write(output); -console.log(`BDV ${claimId}: ${evidence.verdict}; evidence appended to docs/experiments/bdv-pilot/evidence.jsonl`); -process.exitCode = accepted ? 0 : 1; + const after = await captureInputs(); + const inputsStable = preflightPassed && sameInputs(before, after); + const exitCode = result.status ?? 1; + const evaluated = evaluateRun({ + exitCode, + report, + expectedCases: claim.acceptance.expectedCases, + inputsStable, + }); + const durationMs = Math.round(performance.now() - started); + const structuredResult = { + totalTests: report?.numTotalTests ?? null, + passedTests: report?.numPassedTests ?? null, + failedTests: report?.numFailedTests ?? null, + scenarioResults: evaluated.scenarioResults, + }; + const output = normalizeOutput(`${result.stdout ?? ''}${result.stderr ?? ''}`, tempDirectory); + const evidence = { + schemaVersion: 2, + claimId, + claimDigest: before.claimDigest, + approvedProbeDigest: claim.approvedProbeDigest, + probeDigest: before.probeDigest, + runnerDigest: before.runnerDigest, + commit: before.commit, + commitAfter: after.commit, + claimDigestAfter: after.claimDigest, + probeDigestAfter: after.probeDigest, + runnerDigestAfter: after.runnerDigest, + inputsStable, + dirtyPathsBefore: before.dirtyPaths, + dirtyPathsAfter: after.dirtyPaths, + startedAt: startedAt.toISOString(), + durationMs, + timeoutMs, + timedOut: result.error?.code === 'ETIMEDOUT', + command: command.map((part) => (part.startsWith(tempDirectory) ? '/vitest-report.json' : part)), + exitCode, + ...structuredResult, + resultDigest: digest(JSON.stringify(structuredResult)), + verdict: evaluated.accepted ? 'pass' : 'fail', + outputDigest: digest(output), + output, + error: result.error ? { code: result.error.code, message: result.error.message } : null, + }; + + await mkdir(path.dirname(evidenceFile), { recursive: true }); + await appendFile(evidenceFile, `${JSON.stringify(evidence)}\n`); + process.stdout.write(output); + console.log(`BDV ${claimId}: ${evidence.verdict}; evidence appended to ${evidencePath}`); + if (!preflightPassed) { + console.error( + `BDV refused to run: ${ + !approvedProbe + ? 'probe digest differs from the frozen contract' + : 'working tree is dirty or Git state is unavailable' + }`, + ); + } else if (!inputsStable) { + console.error('BDV rejected the run because executable inputs changed during the probe'); + } + process.exitCode = evaluated.accepted ? 0 : 1; +} + +if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) { + await main(); +} From 9cf8c6a950ce3024bc59bc3788b0a5f07d67489e Mon Sep 17 00:00:00 2001 From: qduc Date: Thu, 8 Oct 2026 23:06:23 +0700 Subject: [PATCH 4/7] docs(bdv): record seeded-fault control --- docs/experiments/bdv-pilot/evidence.jsonl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/experiments/bdv-pilot/evidence.jsonl b/docs/experiments/bdv-pilot/evidence.jsonl index 8abe085e..44947b03 100644 --- a/docs/experiments/bdv-pilot/evidence.jsonl +++ b/docs/experiments/bdv-pilot/evidence.jsonl @@ -1 +1,4 @@ {"schemaVersion":1,"claimId":"approval-call-binding","claimDigest":"aa52df0624b9888e789272e09bf21d8126663b4802664671afcfffb3788d5d07","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigest":"c8ed6477c63fc2507a7979a6c92db43c89d19c8f48003465626d67808d2e0be6","commit":"0425b5a12e6b4429f3621d16a6f3c38205e0dc5e","startedAt":"2026-10-08T15:45:59.677Z","durationMs":2900,"command":["pnpm","exec","vitest","run","--reporter=minimal","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call"],"exitCode":0,"passedFiles":1,"passedTests":4,"verdict":"pass","outputDigest":"b89ec2955b9af86381110d1be4d22669b178c74c2cc45f149e87728f4a71e16d","output":"\n RUN v4.1.9 \n\n\n Test Files 1 passed (1)\n Tests 4 passed | 30 skipped (34)\n Start at 22:45:59\n Duration 2.59s (transform 31ms, setup 12ms, import 36ms, tests 2.48s, environment 0ms)\n\n"} +{"schemaVersion":2,"claimId":"approval-call-binding","claimDigest":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","approvedProbeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigest":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","commit":"47ccac4811b79e2bba92d7d87123568718f0d227","commitAfter":"47ccac4811b79e2bba92d7d87123568718f0d227","claimDigestAfter":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","probeDigestAfter":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigestAfter":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","inputsStable":true,"dirtyPathsBefore":[],"dirtyPathsAfter":[],"startedAt":"2026-10-08T16:00:07.451Z","durationMs":3002,"timeoutMs":120000,"timedOut":false,"command":["pnpm","exec","vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/vitest-report.json"],"exitCode":0,"totalTests":34,"passedTests":4,"failedTests":0,"scenarioResults":[{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","status":"passed"},{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","status":"passed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","status":"passed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids","status":"passed"}],"resultDigest":"327c214cadb80219c3d2b660ea5200e344705cef172b90a92e2e2644206861f7","verdict":"pass","outputDigest":"3dfac86c28083d2005ef7131a28dcc65fe54a0bd51fce1c6c61a431bea5a2fa7","output":"JSON report written to /vitest-report.json\n","error":null} +{"schemaVersion":2,"claimId":"approval-call-binding","claimDigest":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","approvedProbeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigest":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","commit":"47ccac4811b79e2bba92d7d87123568718f0d227","commitAfter":"47ccac4811b79e2bba92d7d87123568718f0d227","claimDigestAfter":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","probeDigestAfter":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigestAfter":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","inputsStable":false,"dirtyPathsBefore":[".bdv-dirty-input-probe"],"dirtyPathsAfter":[".bdv-dirty-input-probe"],"startedAt":"2026-10-08T16:00:40.098Z","durationMs":39,"timeoutMs":120000,"timedOut":false,"command":["pnpm","exec","vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/vitest-report.json"],"exitCode":1,"totalTests":null,"passedTests":null,"failedTests":null,"scenarioResults":[{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","status":"missing"},{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","status":"missing"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","status":"missing"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids","status":"missing"}],"resultDigest":"4e05a53322a157c8f07503e5ab538b4e253fb931fefd66609e547aa7b3c8ba52","verdict":"fail","outputDigest":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","output":"","error":null} +{"schemaVersion":2,"recordType":"seeded-fault-control","claimId":"approval-call-binding","baseCommit":"6ccef47824af7b3387c651c04e43395bb088d8f5","approvedProbeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","fault":"ApprovalLedger.blanketDecision treats any nonempty per-call approved list as blanket approval","faultPatchDigest":"14dfd2e9ed15f1a2e48711e6365f674779901bd71284018a2c38ceff616fa300","command":["CI=1","node_modules/.bin/vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/private/tmp/bdv-seeded-fault-report.json"],"exitCode":1,"totalTests":34,"passedTests":0,"failedTests":4,"failingScenarios":["an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids"],"observedFailure":"all four scenarios removed victim.txt instead of preserving its contents; assertion expected 'must survive\\n' but received null","reportDigest":"1d42575f8391b62fb5dd751f9cb5a21952dac3b565986794f15890dda824eb7c","verdict":"expected-fail"} From 1c51842882cbd2ea1d2ca828913a34a39776919d Mon Sep 17 00:00:00 2001 From: qduc Date: Fri, 9 Oct 2026 07:32:19 +0700 Subject: [PATCH 5/7] docs(bdv): refreeze probe after rebase --- docs/experiments/bdv-pilot/claims.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/experiments/bdv-pilot/claims.json b/docs/experiments/bdv-pilot/claims.json index 44ee7a87..d64d0a3b 100644 --- a/docs/experiments/bdv-pilot/claims.json +++ b/docs/experiments/bdv-pilot/claims.json @@ -6,7 +6,7 @@ "statement": "An approval for one shell tool call cannot authorize a later call that reuses the same provider call id, and adapter-generated ids for missing or empty provider ids remain distinct.", "authority": "Frozen pilot contract derived from the existing user-facing approval policy and the expected safe side effects; this runner cannot alter the expected outcomes.", "probe": "approval-reuse-integration", - "approvedProbeDigest": "05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328", + "approvedProbeDigest": "c587fc30c0efe640d83a89f2ece92771406673c67102beca40ce84b3db3afb7c", "acceptance": { "expectedCases": [ "an approved shell call does not approve a later call that reuses its id when the provider repeats call_0", From ee0d5cda571f23e47d721486fa4c2b20cc1c00af Mon Sep 17 00:00:00 2001 From: qduc Date: Fri, 9 Oct 2026 07:33:30 +0700 Subject: [PATCH 6/7] test(cli): match bounded shell command previews --- docs/experiments/bdv-pilot/claims.json | 2 +- source/cli.integration.test.ts | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/experiments/bdv-pilot/claims.json b/docs/experiments/bdv-pilot/claims.json index d64d0a3b..d83147dd 100644 --- a/docs/experiments/bdv-pilot/claims.json +++ b/docs/experiments/bdv-pilot/claims.json @@ -6,7 +6,7 @@ "statement": "An approval for one shell tool call cannot authorize a later call that reuses the same provider call id, and adapter-generated ids for missing or empty provider ids remain distinct.", "authority": "Frozen pilot contract derived from the existing user-facing approval policy and the expected safe side effects; this runner cannot alter the expected outcomes.", "probe": "approval-reuse-integration", - "approvedProbeDigest": "c587fc30c0efe640d83a89f2ece92771406673c67102beca40ce84b3db3afb7c", + "approvedProbeDigest": "3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061", "acceptance": { "expectedCases": [ "an approved shell call does not approve a later call that reuses its id when the provider repeats call_0", diff --git a/source/cli.integration.test.ts b/source/cli.integration.test.ts index 244dd017..69bae14a 100644 --- a/source/cli.integration.test.ts +++ b/source/cli.integration.test.ts @@ -11,6 +11,11 @@ import { createTestChildEnv } from './test-helpers/terminal-e2e.js'; const require = createRequire(import.meta.url); +function expectedToolLine(command: string): string { + const preview = command.length > 80 ? `${command.slice(0, 80)}…` : command; + return `[tool] shell: ${preview}`; +} + // dist/ is a gitignored build artifact, so a fresh worktree has no compiled // CLI and these tests would fail until someone runs `pnpm build`. Instead of // spawning tsx per test (which pays a ~2s compile per process), compile the @@ -1327,10 +1332,7 @@ function expectSecondCallRefused(run: Awaited `[tool] shell: ${command}`)); + expect(run.toolLines, report).toEqual(run.commands.map(expectedToolLine)); // The saved session log (what resume replays) records both executions and both outputs, once each. expect(run.savedToolStarts).toEqual(run.commands); expect(run.savedCommandOutputs).toEqual(run.commands); @@ -1415,7 +1417,7 @@ it('one approved shell execution prints exactly one [tool] line', { timeout: 90_ expect(run.status, report).toBe(0); expect(run.files, report).toEqual({ 'one.txt': 'one\n' }); expect(run.stderr.match(/\[approval required\] shell/g), report).toHaveLength(1); - expect(run.toolLines, report).toEqual([`[tool] shell: ${run.commands[0]}`]); + expect(run.toolLines, report).toEqual([expectedToolLine(run.commands[0])]); expect(run.savedToolStarts).toEqual(run.commands); expect(run.savedCommandOutputs).toEqual(run.commands); }); @@ -1437,7 +1439,7 @@ it('a call that reuses a refused call id runs and is saved as run, not as refuse const report = `exit ${run.status}\nstderr:\n${run.stderr}`; expect(run.status, report).toBe(0); expect(run.files, report).toEqual({ 'two.txt': 'two\n', 'victim.txt': 'must survive\n' }); - expect(run.toolLines, report).toEqual([`[tool] shell: ${refusedCommand}`, `[tool] shell: ${ranCommand}`]); + expect(run.toolLines, report).toEqual([expectedToolLine(refusedCommand), expectedToolLine(ranCommand)]); // Only the first call was refused. expect(run.stderr.match(/Approval Rejected:/g), report).toHaveLength(1); expect(run.stderr, report).toContain(`cannot be executed automatically: ${refusedCommand}`); From 431b55ff7e3e78df48be2a6bcb6ce09e4c43837a Mon Sep 17 00:00:00 2001 From: qduc Date: Fri, 9 Oct 2026 07:34:39 +0700 Subject: [PATCH 7/7] docs(bdv): verify rebased probe behavior --- docs/experiments/bdv-pilot/evidence.jsonl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/experiments/bdv-pilot/evidence.jsonl b/docs/experiments/bdv-pilot/evidence.jsonl index 44947b03..38e236e6 100644 --- a/docs/experiments/bdv-pilot/evidence.jsonl +++ b/docs/experiments/bdv-pilot/evidence.jsonl @@ -2,3 +2,6 @@ {"schemaVersion":2,"claimId":"approval-call-binding","claimDigest":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","approvedProbeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigest":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","commit":"47ccac4811b79e2bba92d7d87123568718f0d227","commitAfter":"47ccac4811b79e2bba92d7d87123568718f0d227","claimDigestAfter":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","probeDigestAfter":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigestAfter":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","inputsStable":true,"dirtyPathsBefore":[],"dirtyPathsAfter":[],"startedAt":"2026-10-08T16:00:07.451Z","durationMs":3002,"timeoutMs":120000,"timedOut":false,"command":["pnpm","exec","vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/vitest-report.json"],"exitCode":0,"totalTests":34,"passedTests":4,"failedTests":0,"scenarioResults":[{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","status":"passed"},{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","status":"passed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","status":"passed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids","status":"passed"}],"resultDigest":"327c214cadb80219c3d2b660ea5200e344705cef172b90a92e2e2644206861f7","verdict":"pass","outputDigest":"3dfac86c28083d2005ef7131a28dcc65fe54a0bd51fce1c6c61a431bea5a2fa7","output":"JSON report written to /vitest-report.json\n","error":null} {"schemaVersion":2,"claimId":"approval-call-binding","claimDigest":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","approvedProbeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigest":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","commit":"47ccac4811b79e2bba92d7d87123568718f0d227","commitAfter":"47ccac4811b79e2bba92d7d87123568718f0d227","claimDigestAfter":"59b62df2e980bb6ead1da97256622f75382e6360ec1c6b8a2f316159ce552008","probeDigestAfter":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","runnerDigestAfter":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","inputsStable":false,"dirtyPathsBefore":[".bdv-dirty-input-probe"],"dirtyPathsAfter":[".bdv-dirty-input-probe"],"startedAt":"2026-10-08T16:00:40.098Z","durationMs":39,"timeoutMs":120000,"timedOut":false,"command":["pnpm","exec","vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/vitest-report.json"],"exitCode":1,"totalTests":null,"passedTests":null,"failedTests":null,"scenarioResults":[{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","status":"missing"},{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","status":"missing"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","status":"missing"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids","status":"missing"}],"resultDigest":"4e05a53322a157c8f07503e5ab538b4e253fb931fefd66609e547aa7b3c8ba52","verdict":"fail","outputDigest":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","output":"","error":null} {"schemaVersion":2,"recordType":"seeded-fault-control","claimId":"approval-call-binding","baseCommit":"6ccef47824af7b3387c651c04e43395bb088d8f5","approvedProbeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","probeDigest":"05ba6586bc09333bb248e7487f69c4598cc180eb4fccd3d65bb3b2a8922d3328","fault":"ApprovalLedger.blanketDecision treats any nonempty per-call approved list as blanket approval","faultPatchDigest":"14dfd2e9ed15f1a2e48711e6365f674779901bd71284018a2c38ceff616fa300","command":["CI=1","node_modules/.bin/vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/private/tmp/bdv-seeded-fault-report.json"],"exitCode":1,"totalTests":34,"passedTests":0,"failedTests":4,"failingScenarios":["an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids"],"observedFailure":"all four scenarios removed victim.txt instead of preserving its contents; assertion expected 'must survive\\n' but received null","reportDigest":"1d42575f8391b62fb5dd751f9cb5a21952dac3b565986794f15890dda824eb7c","verdict":"expected-fail"} +{"schemaVersion":2,"claimId":"approval-call-binding","claimDigest":"27afbbe942d3b4da4896418f7114ab707abdbc3647f85343daed90f3d243e82e","approvedProbeDigest":"c587fc30c0efe640d83a89f2ece92771406673c67102beca40ce84b3db3afb7c","probeDigest":"c587fc30c0efe640d83a89f2ece92771406673c67102beca40ce84b3db3afb7c","runnerDigest":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","commit":"1c51842882cbd2ea1d2ca828913a34a39776919d","commitAfter":"1c51842882cbd2ea1d2ca828913a34a39776919d","claimDigestAfter":"27afbbe942d3b4da4896418f7114ab707abdbc3647f85343daed90f3d243e82e","probeDigestAfter":"c587fc30c0efe640d83a89f2ece92771406673c67102beca40ce84b3db3afb7c","runnerDigestAfter":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","inputsStable":true,"dirtyPathsBefore":[],"dirtyPathsAfter":[],"startedAt":"2026-10-09T00:32:22.912Z","durationMs":3440,"timeoutMs":120000,"timedOut":false,"command":["pnpm","exec","vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/vitest-report.json"],"exitCode":1,"totalTests":37,"passedTests":0,"failedTests":4,"scenarioResults":[{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","status":"failed"},{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","status":"failed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","status":"failed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids","status":"failed"}],"resultDigest":"ecd2b6913729ad3adaf2786819612bcffffb7c6dc968518fabf19400d8974374","verdict":"fail","outputDigest":"3dfac86c28083d2005ef7131a28dcc65fe54a0bd51fce1c6c61a431bea5a2fa7","output":"JSON report written to /vitest-report.json\n","error":null} +{"schemaVersion":2,"claimId":"approval-call-binding","claimDigest":"be535d04c03fbea903ffa6542e20f005b2173249043201fcdb4acd0986eade07","approvedProbeDigest":"3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061","probeDigest":"3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061","runnerDigest":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","commit":"ee0d5cda571f23e47d721486fa4c2b20cc1c00af","commitAfter":"ee0d5cda571f23e47d721486fa4c2b20cc1c00af","claimDigestAfter":"be535d04c03fbea903ffa6542e20f005b2173249043201fcdb4acd0986eade07","probeDigestAfter":"3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061","runnerDigestAfter":"bdb45c902b50b38253253eda2fff82539dbf1f9bb1b78b586f76e60b7655392c","inputsStable":true,"dirtyPathsBefore":[],"dirtyPathsAfter":[],"startedAt":"2026-10-09T00:33:36.674Z","durationMs":2955,"timeoutMs":120000,"timedOut":false,"command":["pnpm","exec","vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/vitest-report.json"],"exitCode":0,"totalTests":37,"passedTests":4,"failedTests":0,"scenarioResults":[{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","status":"passed"},{"fullName":"an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","status":"passed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","status":"passed"},{"fullName":"an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids","status":"passed"}],"resultDigest":"25bb51501ca6c75f14849069f72d031eccc40cdddf499c9a54846dc1582a29a0","verdict":"pass","outputDigest":"3dfac86c28083d2005ef7131a28dcc65fe54a0bd51fce1c6c61a431bea5a2fa7","output":"JSON report written to /vitest-report.json\n","error":null} +{"schemaVersion":2,"recordType":"seeded-fault-control","claimId":"approval-call-binding","baseCommit":"ee0d5cda571f23e47d721486fa4c2b20cc1c00af","approvedProbeDigest":"3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061","probeDigest":"3cc8df47576f3754b4c5d6b0ac6a75870e72336b8608cee6ced2fbd60bb59061","fault":"ApprovalLedger.blanketDecision treats any nonempty per-call approved list as blanket approval","faultPatchDigest":"14dfd2e9ed15f1a2e48711e6365f674779901bd71284018a2c38ceff616fa300","command":["CI=1","node_modules/.bin/vitest","run","--reporter=json","--config","vitest.integration.config.ts","source/cli.integration.test.ts","-t","an approved shell call does not approve a later call","--outputFile","/private/tmp/bdv-rebased-seeded.json"],"exitCode":1,"totalTests":37,"passedTests":0,"failedTests":4,"failingScenarios":["an approved shell call does not approve a later call that reuses its id when the provider repeats call_0","an approved shell call does not approve a later call that reuses its id when the provider repeats its own id","an approved shell call does not approve a later call when the chat-completions provider omits tool call ids","an approved shell call does not approve a later call when the chat-completions provider sends empty tool call ids"],"observedFailure":"each scenario deleted victim.txt instead of preserving 'must survive\\n'","reportDigest":"04f33822254120c6bb8248c36c2e7c3d2fbb20dea7ec83756d84dd1ed893aa8e","verdict":"expected-fail"}