diff --git a/.gitattributes b/.gitattributes index 60c3b54..27016d2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,6 @@ # Check every text file out with LF on all platforms. Generated artifacts embed the bytes of # descriptors, profiles, and docs verbatim, and the byte-exact tests compare against LF fixtures, so -# a CRLF working tree on Windows silently changes program output. +# line-ending conversion would silently change program output. * text=auto eol=lf # Golden fixtures are byte-exact — never EOL-normalize them. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 191bf69..7e7f4e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,13 +16,7 @@ env: jobs: test: name: Test suite - strategy: - # Windows is the platform this matrix exists to cover, so a Linux failure - # must not cancel it — that is exactly when its result is worth having. - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install Rust toolchain @@ -31,33 +25,14 @@ jobs: components: rustfmt, clippy - name: Cache cargo build uses: Swatinem/rust-cache@v2 - - name: Install jq - # Git for Windows supplies sh, xargs, tr, and wc, but not jq, and the - # judge-recipe tests execute the shipped pipeline text rather than a - # stand-in for it. - if: runner.os == 'Windows' - run: choco install jq --yes --no-progress - - name: Permit symlink creation - # Windows creates symlinks only under Developer Mode or elevation, and - # the core::fs round-trips need one. Asking for it explicitly beats - # depending on how the runner's token happens to be built. - if: runner.os == 'Windows' - run: > - reg add "HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock" - /t REG_DWORD /f /v AllowDevelopmentWithoutDevLicense /d 1 - name: Format check run: cargo fmt --all -- --check - name: Clippy run: cargo clippy --all-targets --all-features -- -D warnings - name: Test - # Capability-gated tests (the POSIX recipe pipelines, symlink - # round-trips, long-path staging) skip with a printed reason on a host - # that lacks the capability. This turns every such skip into a failure, - # so neither runner can quietly stop covering them. Ubuntu ships the - # recipe tools; the steps above provide them on Windows. Long paths need - # no provisioning — the runner passes core.longpaths to git itself. - # EVAL_MAGIC_SH stays unset deliberately: discovering the shell from the - # Git install root is what a Windows user hits, so CI should run it too. + # Capability-gated tests skip with a printed reason on a host that lacks + # the capability. CI turns every such skip into a failure so coverage + # cannot shrink silently. env: EVAL_MAGIC_REQUIRE_POSIX_TOOLS: 1 run: cargo test --all-targets diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ea2445..1f4be2b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -113,9 +113,6 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json steps: - - name: enable windows longpaths - run: | - git config --global core.longpaths true - uses: actions/checkout@v6 with: persist-credentials: false @@ -146,9 +143,7 @@ jobs: echo "dist ran successfully" - id: cargo-dist name: Post-build - # We force bash here just because github makes it really hard to get values up - # to "real" actions without writing to env-vars, and writing to env-vars has - # inconsistent syntax between shell and powershell. + # Force bash so every release runner writes GitHub outputs with one syntax. shell: bash run: | # Parse out what we just built and upload it to scratch storage diff --git a/AGENTS.md b/AGENTS.md index b2d4b94..6a1abb9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,12 +5,12 @@ `eval-magic` is a Rust CLI crate. The binary entry point is `src/main.rs`; reusable logic lives in `src/lib.rs` and submodules such as `cli/`, `pipeline/`, `sandbox/`, `validation/`, and `workspace/`. JSON schemas are tracked in `schema/`, harness descriptors (plus embedded harness -assets such as the OpenCode write-guard plugin template) in `harnesses/`, shared -prompt profiles in `profiles/`, and docs in `docs/` — user-facing Markdown under `docs/guides/` -ships embedded in the binary as `eval-magic docs `; the other files are internal development -docs. `docs/developer_overview.md` maps the repository and holds the placement policy. Integration -tests are split by surface area under `tests/cli/` and `tests/run/`; unit tests usually live beside -the module they exercise. +assets such as the OpenCode write-guard plugin template) in `harnesses/`, the `RUNBOOK.md` prose +template in `profiles/`, and docs in `docs/` — user-facing Markdown under `docs/guides/` ships +embedded in the binary as `eval-magic docs `; the other files are internal development docs. +`docs/developer_overview.md` maps the repository and holds the placement policy. Integration tests +are split by surface area under `tests/cli/` and `tests/run/`; unit tests usually live beside the +module they exercise. ## Build, Test, and Development Commands @@ -54,41 +54,35 @@ Extraction is a size decision, not a style preference; don't split a small inlin **Spawning a child process from a test.** Use the hidden `__fixture` subcommand, never `sh`, `true`, `printf`, or a `#!/bin/sh` stub. It exits with a chosen code, emits chosen bytes, writes a chosen file, or checks a file or variable — see `FixtureArgs` in `src/cli/args.rs`. One invocation parses -the same under `sh -c` and `cmd /C`, which is what keeps `command_check` tests off per-OS command -strings. Build the command with the `fixture` helper (`tests/run/helpers.rs` for integration tests, -the one in `src/pipeline/grade/command_check/tests.rs` for unit tests). Because the fixture is the -binary, `cargo test --lib` alone does not build it — run `cargo test`, or `cargo build` first. - -**Tests are gated on capabilities, not on the OS.** `#[cfg(unix)]` on a test hides it from -compilation and clippy on the other host and hides the coverage gap. Instead, probe for what the -test actually needs and call `report_skip` (`src/core/runtime.rs`), which prints the reason and -returns `true`. Setting `EVAL_MAGIC_REQUIRE_POSIX_TOOLS=1` turns every skip into a failure; CI sets -it on both runners, so neither can quietly stop covering something. Three capabilities are gated -today: the recipe tools beyond the shell itself (`require_posix_toolchain` — in practice `jq`), -symlink creation, which Windows allows only under Developer Mode, and creating a path past -Windows' 259-character limit (`deep_task_root`, `src/cli/run/orchestrate/git.rs`). The Windows -runner is provisioned for those rather than exempted from them, so a skip there is a red build. The -shell is not one of them; it is a hard requirement, per the section below. -`require_posix_toolchain` is not test-only either — the `run` preflight uses it to warn about the -same gap. Where a genuine per-OS difference is the behavior under test — signals, path separators — -branch on `cfg!(windows)` at runtime so both arms still compile everywhere. +predictably under `sh -c`, which keeps `command_check` tests focused on runner behavior instead of +the host's utility implementations. Build the command with the `fixture` helper +(`tests/run/helpers.rs` for integration tests, the one in +`src/pipeline/grade/command_check/tests.rs` for unit tests). Because the fixture is the binary, +`cargo test --lib` alone does not build it — run `cargo test`, or `cargo build` first. + +**Tests are gated on capabilities, not on broad platform labels.** Probe for what the test actually +needs and call `report_skip` (`src/core/runtime.rs`), which prints the reason and returns `true`. +Setting `EVAL_MAGIC_REQUIRE_POSIX_TOOLS=1` turns every skip into a failure; CI sets it so coverage +cannot quietly shrink. Symlink creation is capability-gated because the backing filesystem may +forbid it. The shell is not capability-gated; it is a hard requirement, per the section below. **A POSIX shell is required, for use and for development.** Harness `exec_template`s are POSIX -command lines, so the dispatch and probe paths spawn `sh` via `posix_shell()` -(`src/core/runtime.rs`) rather than a hardcoded `/bin/sh`: it searches `PATH`, then a Git for -Windows install. Set `EVAL_MAGIC_SH` to override it. `cargo test` inherits the requirement — the -scripted-turn tests spawn a `#!/bin/sh` harness stub through the resolved shell and do not skip — -so a host without `sh` fails the suite instead of quietly covering less. `jq` is required alongside -it for the parallel-dispatch and judge recipes; Git for Windows supplies the shell, `xargs`, `tr`, -and `wc`, but not `jq`. `POSIX_TOOLING_REQUIREMENT` (`src/core/runtime.rs`) is the one wording the +command lines, so the dispatch and probe paths spawn `sh` through `run_in_posix_shell` / +`posix_shell()` (`src/core/runtime.rs`) rather than a hardcoded `/bin/sh`: it searches `PATH`, then +checks `/bin/sh`. Set `EVAL_MAGIC_SH` to override it. `cargo test` inherits the requirement — the +dispatch tests spawn a `#!/bin/sh` harness stub through the resolved shell and do not skip — so a +host without `sh` fails the suite instead of quietly covering less. The shell is the whole +requirement: `jq` was needed only while operators pasted the generated dispatch and judge recipes, +and `eval-magic dispatch` drives both itself. +`POSIX_TOOLING_REQUIREMENT` (`src/core/runtime.rs`) is the one wording the Markdown-carrying surfaces reuse: the shell-discovery errors, the `run` preflight warnings, `RUNBOOK.md`, and `dispatch-manifest.md`. State the requirement from there rather than rephrasing it. `--help` is the one deliberate restatement (`AFTER_HELP` in `src/cli/help.rs`), hard-wrapped and backtick-free because clap renders into a terminal; keep the two in step by hand. -Which platforms that requirement is honored on — and why preparing on Windows but dispatching from -WSL is a correctness boundary rather than a preference — is stated once under "Platform support" in -`docs/developer_overview.md`. +eval-magic supports Linux and macOS. On Windows, use and develop eval-magic entirely inside WSL; +native Windows is unsupported. The complete boundary and the portable-data exception are stated +under "Platform support" in `docs/developer_overview.md`. **Where user-facing warnings come from.** Library modules (`pipeline`, `workspace`, `sandbox`, `adapters`) never print. They return warning strings on their result struct — `#[serde(skip)]` when diff --git a/Cargo.lock b/Cargo.lock index 4c67cff..032faa7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -275,7 +275,7 @@ dependencies = [ [[package]] name = "eval-magic" -version = "0.9.1" +version = "0.10.0" dependencies = [ "anyhow", "assert_cmd", @@ -287,11 +287,9 @@ dependencies = [ "regex", "serde", "serde_json", - "similar", "tempfile", "thiserror", "toml", - "walkdir", ] [[package]] @@ -881,15 +879,6 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -949,12 +938,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "similar" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6505efef05804732ed8a3f2d4f279429eb485bd69d5b0cc6b19cc02005cda16" - [[package]] name = "smallvec" version = "1.15.1" @@ -1137,16 +1120,6 @@ dependencies = [ "libc", ] -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "wasip2" version = "1.0.3+wasi-0.2.9" @@ -1201,15 +1174,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys", -] - [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index ba06307..01d1a70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "eval-magic" -version = "0.9.1" +version = "0.10.0" edition = "2024" description = "One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior." license = "MIT" @@ -38,20 +38,18 @@ clap = { version = "4.6.1", features = ["derive"] } # schemas are embedded (include_str!) and use only internal #/definitions refs, # so no resolver is needed at all. jsonschema = { version = "0.46.5", default-features = false } -# Bash write-mutation patterns (sandbox policy). default-features off drops the -# Unicode tables: the shell-command patterns are ASCII, and ASCII `\b`/`\s` is -# the intended semantics, so `perf` (matching speed) is all we keep beyond `std`. +# Eval validation, transcript checks, dispatch parsing, and staging patterns. +# Default features are disabled to drop Unicode tables: these authored patterns +# use ASCII syntax, so `perf` (matching speed) is all we keep beyond `std`. regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.150", features = ["preserve_order"] } -similar = { version = "3.1.1", default-features = false } tempfile = "3.27.0" thiserror = "2.0.18" # Harness descriptor files (harnesses/*.toml). `display` serializes the # resolved (layer-merged) descriptor back to authorable TOML for # `harness show`. toml = { version = "0.9", default-features = false, features = ["parse", "serde", "display"] } -walkdir = "2.5.0" [dev-dependencies] assert_cmd = "2.2.2" diff --git a/README.md b/README.md index 65b8759..1c2eb49 100644 --- a/README.md +++ b/README.md @@ -28,41 +28,31 @@ eval-magic runs the same task in two controlled conditions—such as a new skill versus no skill, or an edited skill versus its previous version—and grades both results against shared assertions. It -builds isolated task workspaces, stages skills, generates harness-specific dispatch instructions, -ingests transcripts and final state, and produces comparison artifacts. You dispatch the agent -sessions with Claude Code, Cline, Codex, OpenCode, or a descriptor-backed harness of your own. +builds isolated task workspaces, stages skills, dispatches the agent sessions itself, ingests +transcripts and final state, and produces comparison artifacts. It drives Claude Code, Cline, +Codex, OpenCode, or a descriptor-backed harness of your own. The installed CLI is the primary manual. Start with `eval-magic --help`, and use `eval-magic --help` whenever you reach a new phase. ## Install -Git is required at runtime, plus a POSIX shell with `jq`: the dispatch and judge recipes eval-magic -generates are POSIX command lines built on `jq`, `xargs`, `tr`, and `wc`. The shell that runs them -has to resolve the same paths the workspace was prepared with. On Windows that is Git Bash (Git for -Windows), with `jq` installed separately — Git for Windows does not bundle it. WSL resolves a -different filesystem namespace, so run eval-magic inside WSL rather than dispatching into it. -Set `EVAL_MAGIC_SH` to select a specific `sh`. +eval-magic supports Linux and macOS. On Windows, install and run eval-magic inside Windows +Subsystem for Linux (WSL); native Windows is unsupported. Keep the repository, workspace, and +harness commands inside the same WSL environment. -Windows support runs through Git Bash and is deprecated: a future release will require WSL. +Git and a POSIX shell are required. Set `EVAL_MAGIC_SH` to select a specific `sh`. -Prebuilt binaries for macOS, Linux, and Windows are attached to each +Prebuilt binaries for macOS and Linux are attached to each [GitHub release](https://github.com/slowdini/eval-magic/releases). -macOS or Linux: +Install on macOS, Linux, or inside WSL: ```bash curl --proto '=https' --tlsv1.2 -LsSf \ https://github.com/slowdini/eval-magic/releases/latest/download/eval-magic-installer.sh | sh ``` -Windows PowerShell: - -```powershell -powershell -ExecutionPolicy Bypass -c \ - "irm https://github.com/slowdini/eval-magic/releases/latest/download/eval-magic-installer.ps1 | iex" -``` - Or build and install from crates.io: ```bash @@ -80,8 +70,11 @@ cd path/to/my-skill eval-magic init ``` -`init` creates `evals/evals.json` with one valid seed case. Edit the prompt and expected behavior to -describe a realistic task, add concrete assertions as the eval matures, then check the file: +`init` creates `evals/evals.json` with one valid seed case and the pinned Weeknight example +codebase. Use `eval-magic init --help` to select another URL, local path, or the current directory, +and use `eval-magic docs codebase` for project selection and provenance details. Edit the prompt +and expected behavior to describe a realistic task, add concrete assertions as the eval matures, +then check the file: ```bash eval-magic validate @@ -101,10 +94,10 @@ eval-magic run --harness codex eval-magic run --harness opencode ``` -`run` prepares the campaign; it does not dispatch agents. Review the printed task and model-usage -summary before continuing. Then read the generated `RUNBOOK.md` from beginning to end. It contains -the exact dispatch, ingest, judge, finalize, and `eval-magic teardown` commands for that campaign -and harness. +`run` prepares the campaign; `eval-magic dispatch` runs it. Review the printed task and model-usage +summary before continuing — dispatch is where model usage is spent. Then read the generated +`RUNBOOK.md` from beginning to end. It contains the exact dispatch, ingest, judge, finalize, and +`eval-magic teardown` commands for that campaign and harness. After finalization, open the generated `benchmark.json` to compare pass rates, token and duration measurements, and validity warnings. Use `eval-magic aggregate --help` when you need to combine @@ -120,12 +113,19 @@ eval-magic run --mode revision The command help and generated runbook describe baseline selection and the rest of the workflow. +An eval can treat coordinated skills as one treatment by setting `skill_name` to an ordered list. +Pass one listed member with `--skill`; it remains the eval owner and supplies overlay files. See +`eval-magic docs isolation` for the complete configuration, Mode A/B behavior, and provenance. + ## How it works Each eval case runs once per condition and repetition in its own clean Git repository. The two arms -receive the same task and fixtures; only the condition under test changes. Assertions can combine -LLM judgment with runner-owned command checks, transcript checks, and final diff limits. Scripted -`turns` resume one native harness session so follow-up answers remain part of the same conversation. +receive the same codebase, task, and overlays; only the condition under test changes. Assertions can +combine LLM judgment with runner-owned command checks, transcript checks, and final diff limits. +Multi-turn evals resume one native harness session so follow-up answers remain part of the same +conversation, whether the turns are scripted or derived by a responder. An eval can start that +session in the harness's native plan mode and continue in act mode once the plan is approved +(`eval-magic docs conversations`). Most harness features are declared in TOML descriptors. See the current registry and resolved data instead of relying on a static compatibility table: @@ -144,6 +144,9 @@ eval-magic harness show codex - `eval-magic docs isolation` explains how live or installed skill sources can contaminate a comparison and how to verify isolation. Its source is [docs/guides/isolation.md](docs/guides/isolation.md). +- `eval-magic docs guard` explains eval-authored command allowances, packaged defaults, and the + containment checks those allowances cannot bypass. Its source is + [docs/guides/guard.md](docs/guides/guard.md). - [docs/developer_overview.md](docs/developer_overview.md) maps the codebase, sources of truth, verification workflow, and internal documentation. @@ -152,9 +155,11 @@ Issues and planned work are tracked in the ## Development -Development carries the same host requirement as use: a POSIX shell with `jq`. The scripted-turn -tests spawn `#!/bin/sh` harness stubs through the resolved shell and do not skip, so the suite -cannot pass without one. Tests that need `jq` or symlink creation report a skip instead. +Development carries the same host requirement as use: Linux or macOS with a POSIX shell. On +Windows, clone the repository and run the complete toolchain inside WSL; native Windows development +is unsupported. The dispatch tests spawn `#!/bin/sh` harness stubs through the resolved shell and +do not skip, so the suite cannot pass without one. Tests that need symlink creation report a skip +instead. ```bash cargo fmt --check diff --git a/build.rs b/build.rs index 3765cb2..18f0ae1 100644 --- a/build.rs +++ b/build.rs @@ -22,6 +22,50 @@ fn main() { let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); fs::write(out_dir.join("guide_topics.rs"), generated) .expect("failed to write generated guide topic table"); + + embed_profiles( + &manifest_dir.join("guard-profiles"), + &out_dir.join("guard_profiles.rs"), + "PACKAGED_GUARD_PROFILES", + ); + embed_profiles( + &manifest_dir.join("ignore-profiles"), + &out_dir.join("ignore_profiles.rs"), + "PACKAGED_IGNORE_PROFILES", + ); +} + +/// Render one directory of TOML profiles as a `&[(filename, body)]` table the +/// crate `include!`s. Both packaged profile families — guard command policy and +/// tool-ignore targets — ship this way, so a new profile is a new file and +/// nothing else. +fn embed_profiles(profile_dir: &Path, out_file: &Path, table_name: &str) { + println!("cargo:rerun-if-changed={}", profile_dir.display()); + let mut profiles: Vec = fs::read_dir(profile_dir) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", profile_dir.display())) + .map(|entry| entry.expect("failed to read profile entry").path()) + .filter(|path| path.extension().and_then(|value| value.to_str()) == Some("toml")) + .collect(); + profiles.sort(); + assert!( + !profiles.is_empty(), + "{} must contain a TOML profile", + profile_dir.display() + ); + + let mut generated = format!("const {table_name}: &[(&str, &str)] = &[\n"); + for path in profiles { + let name = path + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or_else(|| panic!("profile path is not UTF-8: {}", path.display())); + let body = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", path.display())); + generated.push_str(&format!(" ({name:?}, {body:?}),\n")); + } + generated.push_str("];\n"); + fs::write(out_file, generated) + .unwrap_or_else(|err| panic!("failed to write {}: {err}", out_file.display())); } fn discover_guides(guide_dir: &Path) -> Vec { diff --git a/dist-workspace.toml b/dist-workspace.toml index 9d1acb2..c87d3ea 100644 --- a/dist-workspace.toml +++ b/dist-workspace.toml @@ -7,10 +7,13 @@ members = ["cargo:."] cargo-dist-version = "0.32.0" # CI backends to support ci = "github" +# cargo-dist adds native Windows setup even when no Windows targets remain. +# Keep the Linux/macOS-only workflow as an intentional local customization. +allow-dirty = ["ci"] # The installers to generate for each app -installers = ["shell", "powershell"] +installers = ["shell"] # Target platforms to build apps for (Rust target-triple syntax) -targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"] +targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu"] # Path that installers should place binaries in install-path = "CARGO_HOME" # Where to host releases diff --git a/docs/claude-notes.md b/docs/claude-notes.md index e606af9..1ce3d8e 100644 --- a/docs/claude-notes.md +++ b/docs/claude-notes.md @@ -31,15 +31,17 @@ hook-entry and `hookSpecificOutput` verdict templates) is rendered by the generi `result` event rather than a file. - `` from the same env; the initial `system.session_id` supplies the id. Verified against `claude --help` on 2026-07-24. ## Permission mode -Every dispatch and judge recipe carries `--permission-mode bypassPermissions`. The obvious -alternative, `acceptEdits`, is wrong here: it auto-approves *file edits* but **not Bash**, and -because the recipe detaches stdin (`.md`, which plan mode permits) and closes the turn with the plan in its + final message. That write is the `[plan_mode.plan_file]` signal (`root = "~/.claude/plans"`, + `content_field = "content"`), and its `content` is the plan artifact. +- An agent that tries to edit while planning is refused with `Cannot write to while in plan + mode.`, and the refusal reaches the terminal `result` event's `permission_denials` like any + other. That is the mode working, so `record-runs` attributes a write refused in a planning round + to plan mode and leaves it out of the count `aggregate` warns on (see "Permission denials"). +- Resuming with `claude -p --resume --permission-mode bypassPermissions` reports + `permissionMode: bypassPermissions` in `init`, keeps the same `session_id`, and edits succeed: + the session leaves plan mode. The Claude Code documentation states the converse — a `-p --resume` + stays in plan mode only when `--permission-prompt-tool` is passed and no `--permission-mode` is. +- The plan file lands in the operator's `~/.claude/plans`, as in any session. The write guard and + the stray-write audit allow that root; eval-magic keeps the copy the judge reads as + `outputs/plan.md`. Relaxing the default closes the common case, not the class — a deny rule, a managed setting, or an operator-overridden mode still refuses calls — so refusals are detected and reported rather than @@ -88,10 +119,12 @@ affected task; see [progressive-enhancements.md](progressive-enhancements.md). `outputs/claude-events.jsonl` is the `-p` stream-json stream. `assistant`/`user` events wrap full Anthropic Messages objects (tool-call extraction matches `tool_result` blocks back to their -`tool_use` by id); a terminal `result` event carries the authoritative final text, wall-clock -duration, and token usage — there are no per-line timestamps. `system`, `rate_limit_event`, and -other non-message events are skipped. The transcript exposes Skill-tool invocations, so the -`__skill_invoked` meta-check is deterministic here. +`tool_use` by id); a terminal `result` event carries the authoritative final text, native duration, +and token usage — there are no per-line timestamps. Runner-driven eval duration comes from the +runner's monotonic subprocess measurement; this native duration remains the fallback for historical +completion artifacts. `system`, `rate_limit_event`, and other non-message events are skipped. The +transcript exposes Skill-tool invocations, so the `__skill_invoked` meta-check is deterministic +here. The built-in descriptor uses the named parser for this cross-event summary, selects its denial reader explicitly, and maps the session roster through the generic @@ -128,8 +161,9 @@ Each `claude -p` dispatch loads the user/global plugins and skills from its Clau staging slug prevents an on-disk collision but not runtime discovery — an installed plugin exposing a same-named skill is discoverable in *both* arms, so the control arm is not truly skill-absent. `plugin_shadow.rs` detects this in every comparison environment. The shared shadow policy records -one finding per logical skill in schema-v2 `plugin-shadow.json`, including every affected cell, -canonical/discovery paths, source-specific remediation, and the runtime identifier the agent sees. +one finding per logical skill and source class in schema-v3 `plugin-shadow.json`, including every +affected cell, canonical/discovery paths, source-specific remediation, and the runtime identifier +the agent sees. Claude plugin skills use their namespaced `:` runtime ID, direct live skills retain the logical name, and staged subjects use their staging-directory slug. Direct live duplicates record user-before-project precedence; a staged subject with its distinct slug remains selected. @@ -145,7 +179,7 @@ topic ([isolation guide](guides/isolation.md)). The per-source strings the banne `--setting-sources project,local` drops **all** user-scope discovery, not just `enabledPlugins`: skills under `/skills` are unloaded too. Verified 2026-08-06 by A/B within one campaign — -the judge recipe carries no `--setting-sources` and its capture lists both `~/.claude/skills` +the judge dispatch carries no `--setting-sources` and its capture lists both `~/.claude/skills` entries and every `:` id, while all 48 isolated eval dispatches list neither. Project-local staged skills are independent of installed plugins, so they still load and the @@ -169,3 +203,6 @@ only write boundary a dispatch has, since the session itself runs under `bypassP invokes the hidden `guard` subcommand (**stable on-disk contract — never rename**), which denies via Claude Code's `hookSpecificOutput` JSON shape and stays silent to allow. Both layers fail open. A deny aborts the offending dispatch; `detect-stray-writes` remains the after-the-fact backstop. +The shared cwd-aware policy allows ordinary installs, builds, tests, and in-place edits inside the +task env, while explicit outside destinations, output escapes, repository-routing escapes, and +remote Git mutations remain blocked. diff --git a/docs/cline-notes.md b/docs/cline-notes.md index 56e0405..6426e55 100644 --- a/docs/cline-notes.md +++ b/docs/cline-notes.md @@ -19,7 +19,8 @@ (676-line NDJSON capture exercising `read_files`, `run_commands`, and two `skills` invocations), `cline history --json`, several `--id` resume attempts, and `eval-magic harness lint harnesses/cline.toml --as-builtin --probe --yes` (the live probe - rendered the exec template, dispatched it, and recovered a non-empty `final-message.md`). + rendered the exec template, dispatched it, parsed the event capture, and recovered a non-empty + final response). Against 3.0.53: a second dispatch exercising `editor`, `run_commands`, `read_files`, and `skills` (arg/result shapes below), and a guard spike — a hand-staged `.cline/plugins/` project plugin whose `beforeTool` hook blocked calls, proving headless plugin auto-load, the @@ -67,13 +68,15 @@ the descriptor references. "Probe capture" refers to the observed dispatches des | `guard.armed_message` | see descriptor | prose authored for eval-magic output (same structure as the other built-ins) | | Plugin hook contract | `beforeTool({snapshot, tool, toolCall, input})`; block with `{skip: true, reason}`; 3000ms default hook budget (plugin spawns with a 2s timeout so a hung arbiter fails open); `spawnSync` works from the plugin sandbox | 3.0.53 spike capture + the binary's runtime hook loop; the docs' `tool_call_before`/`fail_closed` vocabulary lags the binary | | `shadow.preflight` | `cline-skills` | 3.0.53 root probe (one uniquely-named skill per candidate root): dispatch cwd's `.cline/skills` read, ancestor's NOT (no project walk), `~/.agents/skills` IS read (and receives `cline skill install` global installs); `$CLINE_DIR` overrides the `~/.cline` default (3.0.53 binary) | -| `dispatch.capture_prefix` | `cline` | chosen name (judge capture files `$response_base.cline-events.jsonl`) | -| `dispatch.exec_template` / `parallel_command_template` | see descriptor | flags from `cline --help` (`--act` from the 3.0.52 binary’s hidden option registration + behavioral write test); `--json` NDJSON stdout and `` sets the dispatch cwd (no shell `cd` needed, unlike Claude Code); staged @@ -90,10 +93,9 @@ the descriptor references. "Probe capture" refers to the observed dispatches des silent-degradation trap as Claude Code's `acceptEdits`, so the flag stays explicit. - Piped stdin is appended to the prompt context (docs CLI overview), so every recipe detaches with ``, same as codex. -- There is no `--output-last-message`: the terminal `run_result` NDJSON event carries the final - text, and the exec template's trailing jq step writes `final-message.md` from the captured - events (the harness probe's final-message contract checks that file; ingest recovers the text - from the events file via `extract.final_text`). +- The terminal `run_result` NDJSON event carries the final text. The harness probe and ingest both + recover it from the event file through the configured transcript parser; no separate completion + file or jq post-pass is needed. - Stream shape (3.0.52): `agent_event` wrappers (`content_start` per streaming chunk for text/reasoning — 634 in the probe — but once per tool call; `content_end` with complete blocks; `usage`; `iteration_start`/`iteration_end`; `done`), plus `hook_event` lifecycle @@ -129,8 +131,11 @@ the descriptor references. "Probe capture" refers to the observed dispatches des `skills` surfaces `skill`), results attached from the paired `content_end` (per-tool shape coercion, `{"error"}` payloads included), final text from `run_result.text`, ordered assistant messages from `content_end` text blocks, token totals from `run_result.usage` (cache reads - subtracted), duration from `run_result.durationMs`. `transcript_check` patterns match the + subtracted), and fallback duration from `run_result.durationMs`. Runner-driven eval duration uses + the runner's monotonic subprocess measurement. `transcript_check` patterns match the `" "` rendering of the *flattened* args (e.g. `run_commands.*"command"`). + `run_commands`, `editor`, and the read tools also answer to every other harness's spelling of + their role, so a pattern written as `Bash` or `Read` grades here too. - **Deterministic `__skill_invoked`**: `surfaces_skill_invocation = true` with `skill_tool = "skills"` / `skill_arg = "skill"` — the parser hoists the slug to top level, so the meta-check grades from the transcript instead of the LLM-judge fallback. @@ -146,12 +151,15 @@ the descriptor references. "Probe capture" refers to the observed dispatches des write/shell classification works. Known blind spot: `read_files`' `files:[{path}]` stays nested, so the live-source-read path branch doesn't fire for it (shell-based read detection still covers `cat`-style reads). -- **Riding documented fallbacks** (the `run` preflight names each): no `[conversation]` - (scripted `turns` evals are rejected). +- **No native conversation resume**: no `[conversation]`, so scripted `turns` and responder evals + are rejected. - **Write guard** (`cline-plugin` engine): `run` auto-arms a staged project plugin at `.cline/plugins/slow-powers-eval-guard/index.js` whose `beforeTool` hook forwards every tool call to `eval-magic guard-hook --harness cline` (`run_commands`' `commands` array joined into one `command` string for the shared arbiter) and returns `{skip: true, reason}` on deny. + The shared cwd-aware policy allows ordinary installs, builds, tests, and in-place edits inside + the task env while denying recognized explicit destinations outside it, output escapes, + repository-routing escapes, and remote Git mutations. Spike-verified on 3.0.53 (all in a throwaway dir, hand-staged plugin): project plugin dirs auto-load in headless one-shot dispatches (a bare `index.js` needs no package.json; a loose `.js` file at the plugins root is IGNORED); the hook context is `{snapshot, tool, toolCall, diff --git a/docs/codex-notes.md b/docs/codex-notes.md index 50c7be4..09ebe3d 100644 --- a/docs/codex-notes.md +++ b/docs/codex-notes.md @@ -29,13 +29,19 @@ alias. - `--sandbox workspace-write` bounds writes to the env. - `--json` streams events to stdout — captured as `outputs/codex-events.jsonl`; stderr goes to `codex-stderr.log` so progress/status text (e.g. stdin notices) stays out of the JSONL. -- `--output-last-message /final-message.md` writes the final-message file the - pipeline reads. +- `--json` is the sole completion source; the configured transcript reader extracts the final + agent message from the captured events. - `` through `codex exec resume `; - `thread.started.thread_id` supplies the id and each round keeps `--json` plus its own - `--output-last-message` capture. Verified against `codex exec resume --help` on 2026-07-24. + `thread.started.thread_id` supplies the id and each round keeps `--json` with its own event + capture. Verified against `codex exec resume --help` on 2026-07-24. +- `codex exec` has **no plan-mode flag** (codex-cli 0.147.0: `codex exec --help` lists none; plan + mode is a TUI collaboration mode), so the descriptor declares no `[plan_mode]` and `run` rejects + plan-mode evals on Codex. The runtime analogue is `--sandbox read-only`: a descriptor that moved + the sandbox flag into a `{mode_args}` slot with `plan_args = " --sandbox read-only"` and + `act_args = " --sandbox workspace-write"` would opt in, at the cost of an agent that is never told + it is planning. ## Model flag @@ -110,9 +116,18 @@ are omitted from tool arguments so `run.json` records command output once, in `r tool invocations: `command_execution`, `file_change`, `web_search`, and MCP items. `thread.started.thread_id` is normalized as the resumable session id, and every completed `agent_message` is preserved in event order for conversation gating. `transcript_check` matches -these parsed items. The JSONL exposes **no deterministic skill-tool -event**, so `transcript_surfaces_skill_invocation()` is false and the `__skill_invoked` meta-check -uses the LLM-judge fallback. +these parsed items, and because the `[tools]` vocabulary declares `command_execution` and +`file_change` beside their Claude-style spellings, a pattern authored against another harness's +tool names grades here through role aliasing. + +The JSONL exposes no dedicated skill-tool event. A bounded Codex CLI 0.152.1 probe instead emitted +a successful `item.completed` `command_execution` whose `command` contained the exact staged +`SKILL.md` path as a literal argument to `sed`; the small synthetic event shape is preserved in +`tests/fixtures/codex/skill-access-0.152.1.jsonl`. The descriptor's +`[transcript.skill_access]` table therefore names `command_execution`, its command and exit-code +arguments, and the accepted read-command basenames. `__skill_invoked` passes locally only when one +of those commands exits zero with the task's exact staged path. A live source, another treatment +member's path, dynamic or partial path text, and response wording do not count. Codex token totals use its blended workload metric: @@ -124,11 +139,16 @@ max(input_tokens + output_tokens - cached_input_tokens, 0) count reasoning. Resumed `turn.completed` usage is cumulative for the native thread; the Codex descriptor therefore uses the final round's total instead of summing round totals. -`codex exec --json` transcripts do not include a native duration or event timestamps, so -`duration_ms` remains `null`. Aggregate reports the missing sample count; a benchmark statistic -with `n: 0` is unavailable, not a measured zero. Ingest does not migrate timing artifacts -automatically. Run `eval-magic ingest --harness codex --iteration --overwrite` to regenerate -them from preserved transcripts when desired. +`codex exec --json` transcripts do not include a native duration or event timestamps. Runner-driven +dispatches still have duration: `eval-magic dispatch` measures monotonic time inside each Codex +subprocess and sums the initial and resumed rounds. `ingest` writes that measurement to +`timing.json` with `duration_source: "runner"`, independently of the transcript-normalized +`token_source`. + +A historical or externally produced `conversation.json` file without the runner measurement still +has no Codex duration to recover, so `duration_ms` remains `null`. Aggregate reports the missing +sample count; a benchmark statistic with `n: 0` is unavailable, not a measured zero. Ingest does not +migrate existing timing artifacts automatically. ### Permission denials @@ -165,7 +185,10 @@ update/delete source, and move destination from that body and resolves relative hook payload's `cwd`. Bash output validation uses a quote-aware lexical scan for `>`, `>>`, `>|`, file-descriptor-prefixed redirects, and `tee`: every literal target must resolve under an allowed root, while dynamic, malformed, or outside targets are blocked. Merely mentioning an allowed root -elsewhere in the command does not scope an unrelated redirect. +elsewhere in the command does not scope an unrelated redirect. The same cwd-aware policy allows +ordinary installs, builds, tests, and in-place edits inside the task env while denying recognized +explicit destinations outside it. Repository-routing escapes and remote Git mutations remain +blocked. Guard installation initializes `.eval-magic-outputs/guard-denials.jsonl` and records its absolute path in the optional marker field `denialLogPath`. Each block appends only timestamp, harness, @@ -181,8 +204,29 @@ The hook invokes the hidden `guard-codex` subcommand ## Running inside Codex itself -User-facing guidance lives in `eval-magic run --help`: staging writes `.agents/skills`, guarded -runs also write -`.codex/hooks.json`, and Codex's default workspace-write sandbox protects those paths, so the -runner may need approval/escalation or an external terminal invocation. Keep that help text in sync -when this changes. +Preparation and dispatch can encounter separate restrictions from an operator Codex session. +During `eval-magic run`, staging writes `.agents/skills` and guarded runs write `.codex/hooks.json`. +The operator's sandbox can protect those paths, so preparation may require outer approval or an +ordinary terminal invocation. + +During `eval-magic dispatch`, the inner Codex process inherits the outer launch environment. +The report in [issue #245, item 1](https://github.com/slowdini/eval-magic/issues/245) describes an +`Operation not permitted` failure inside the operator Codex sandbox while the exact generated +task succeeded outside it with the task's workspace-write sandbox and eval guard intact. +This is a conditional diagnosis: compare the same generated task command with equivalent inputs, +working directory, and configuration. The error text alone does not establish the cause. + +OpenAI's [sandbox documentation](https://learn.chatgpt.com/docs/sandboxing) states that spawned +commands inherit the operator sandbox boundaries. Its +[OS-level implementation notes](https://learn.chatgpt.com/docs/agent-approvals-security#os-level-sandbox) +also describe environments that block namespace, `bwrap`, or `seccomp` operations needed to create +the Linux sandbox. These limits explain why granting another writable directory may be insufficient; +they do not identify which operation caused the reported failure. + +The operational guidance lives in `eval-magic dispatch --help`, `eval-magic docs isolation`, and +Codex-generated `RUNBOOK.md`. Prefer running the generated dispatch command from the ordinary +terminal. Where the operator surface and policy support it, an alternative is approval or escalation +of the **outer launch of `eval-magic dispatch`**, limited to the required workspace and process access. +Keep the task command's `--sandbox workspace-write` and eval guard enabled. Neither weakening the +inner task protections nor adding runtime sandbox detection or a platform-specific bypass is part +of this remedy. diff --git a/docs/developer_overview.md b/docs/developer_overview.md index efe3adc..ec4e783 100644 --- a/docs/developer_overview.md +++ b/docs/developer_overview.md @@ -12,18 +12,23 @@ focused internal notes instead of duplicating their details. ## How an evaluation moves through the system 1. `eval-magic init` scaffolds an eval workspace next to a skill. Eval definitions describe the - task, fixtures, assertions, conditions, run count, and optional scripted follow-up turns. -2. `eval-magic run` validates the configuration, creates isolated task roots, stages the requested - skill condition, snapshots the starting state, and writes `RUNBOOK.md`, `dispatch.json`, and - related campaign artifacts. The generated runbook—not a checked-in recipe—is the authority for - dispatching that particular campaign. -3. An operator or automation dispatches each task with the selected harness. One-shot tasks invoke - the harness once; scripted conversations use `eval-magic dispatch-task` to preserve one native - harness session across turns. + codebase, task, overlay files, assertions, conditions, run count, and — for a multi-turn eval — + either scripted follow-up turns or a responder policy that derives them. +2. `eval-magic run` validates the configuration, resolves and copies the skill under test into the + iteration, creates isolated task roots, stages the requested skill condition from that copy, + snapshots the starting state, and writes `RUNBOOK.md`, `dispatch.json`, and related campaign + artifacts. The iteration lives in the eval home, which defaults outside the skill's own + repository (`workspace_root_from`, `src/core/context.rs`). The generated runbook—not a + checked-in recipe—is the authority for dispatching that particular campaign. +3. `eval-magic dispatch` runs every task through the selected harness, `--jobs` at a time, each in + its own private environment. A scripted conversation resumes one native harness session across + its turns. Each task ends with a `conversation.json`, which is also what a rerun skips on. 4. `eval-magic ingest` reads the harness outputs, transcript evidence, guard denials, and final task state. Runner-owned deterministic checks and diff-scope evidence are collected here. -5. `eval-magic grade` evaluates runner-owned assertions and emits tasks for assertions that require - an LLM. The generated recipes dispatch those judge tasks through the selected harness. +5. `eval-magic grade` evaluates runner-owned assertions, writes one bounded `judge-evidence.md` + per recorded run, and emits one or more tasks for assertions that require an LLM. Every sample + inlines the exact same bundle for its run. `eval-magic dispatch --judges` runs those judge tasks + through the selected harness. 6. `eval-magic finalize` checks that required work is complete and writes the final per-run and benchmark artifacts. `eval-magic aggregate` combines campaigns when a larger comparison is needed. @@ -43,9 +48,17 @@ preconditions, handoffs, and recovery commands. few named capabilities that require harness-specific code. - `src/sandbox/`, `src/workspace/`, and `src/validation/` own task isolation, filesystem/workspace mechanics, and configuration checks. +- `src/source/` resolves a declared source — a git URL and ref, or a local directory — to a commit, + and materializes it as a tree. It knows nothing about what is being sourced, so both the codebase + a task environment is built from and the skills under test resolve through it. - `schema/` contains the JSON schemas for user input and generated artifacts. - `harnesses/` contains built-in descriptors, descriptor scaffolding, and embedded harness assets. -- `profiles/` contains shared prompt profiles. +- `guard-profiles/` contains packaged command-policy defaults discovered and embedded by `build.rs`. +- `ignore-profiles/` contains packaged tool-ignore targets, discovered and embedded the same way: + which ignore file a detected formatter or linter reads, so `run` can hide the framework's own + staged files from a sourced codebase's tooling. Both families detect through + `src/core/tree_profiles.rs`. +- `profiles/` contains the `RUNBOOK.md` prose template, embedded in the binary. - `tests/cli/` covers CLI and packaging contracts; `tests/run/` covers campaign behavior across the run boundary. Focused unit tests normally live beside the implementation. - `docs/guides/` contains Markdown guides embedded in the binary. Other files under `docs/` are @@ -70,22 +83,19 @@ following authorities: | Tier | Platform | Verified by | | --- | --- | --- | -| Supported | Linux, macOS | the `ubuntu-latest` CI job | -| Deprecated | Windows, through Git Bash (Git for Windows) | the `windows-latest` CI job | -| Unsupported | preparing a workspace on Windows and dispatching it from WSL | — | - -Windows support is deprecated in favor of WSL, and its removal is gated on #256, which replaces -the generated POSIX recipes with a runner-driven `eval-magic dispatch`. Until that lands, the -Windows runner stays green and Windows-native behavior is held to the same bar as any other -platform: a Windows failure is a real failure, not an accepted gap. Do not add new Windows-native -accommodation in the meantime. - -The unsupported row is a correctness boundary rather than a preference. A generated recipe carries -the absolute paths of the host that prepared the workspace. Git Bash shares the Windows filesystem, -so those paths resolve; WSL resolves its own namespace, where a `C:\…` path names nothing. Nothing -in the tree translates between the two, so the split fails quietly instead of loudly. -`POSIX_TOOLING_REQUIREMENT` (`src/core/runtime.rs`) is the single wording every user-facing surface -reuses to state this; `src/cli/help.rs` restates it for clap by hand. +| Supported | Linux, macOS, Linux inside WSL | the `ubuntu-latest` CI job | +| Unsupported | native Windows | — | + +Windows users run the Linux build inside Windows Subsystem for Linux (WSL). Keep the binary, +repository, eval workspaces, and harness processes inside the same WSL environment. `dispatch` +passes workspace-owned absolute paths to harness command lines, so crossing from a native Windows +process into WSL would change the filesystem namespace and invalidate those paths. + +Do not add native Windows accommodations or release targets. Preserve support for Windows-shaped +paths only where they are data read from artifacts or transcripts; those portable-data contracts +do not imply native Windows runtime support. `POSIX_TOOLING_REQUIREMENT` (`src/core/runtime.rs`) is +the single wording every user-facing Markdown surface reuses. `src/cli/help.rs` restates it for +clap by hand. ## Make and verify a change @@ -94,11 +104,11 @@ editing. Add a focused failing test at the narrowest useful boundary, implement run the focused test again. Cross-harness changes belong at shared descriptor, runner, or adapter boundaries unless the evidence requires a named harness capability. -Development carries the host requirement the tool itself declares: a POSIX shell with `jq`. The -scripted-turn tests spawn `#!/bin/sh` harness stubs through the resolved shell and do not skip, so -the suite cannot pass without one. Tests needing `jq`, symlink creation, or a path past Windows' -259-character limit report a skip instead; `EVAL_MAGIC_REQUIRE_POSIX_TOOLS=1` turns those skips into -failures, as CI sets it to do on both its Ubuntu and its Windows runner. +Development requires Linux or macOS with a POSIX shell. Windows contributors clone the repository +and run the complete toolchain inside WSL; native Windows development is unsupported. The dispatch +tests spawn `#!/bin/sh` harness stubs through the resolved shell and do not skip, so the suite +cannot pass without one. Tests needing symlink creation report a skip instead; +`EVAL_MAGIC_REQUIRE_POSIX_TOOLS=1` turns those skips into failures in CI. Before handing work off, run: @@ -143,3 +153,11 @@ implementation evidence in an internal note. `eval-magic docs byoh`. - [Shipped isolation guide](guides/isolation.md) is the repository source for `eval-magic docs isolation`. +- [Shipped codebase guide](guides/codebase.md) is the repository source for + `eval-magic docs codebase`. +- [Shipped guard guide](guides/guard.md) is the repository source for + `eval-magic docs guard`. +- [Shipped conversations guide](guides/conversations.md) is the repository source for + `eval-magic docs conversations`. +- [Shipped judging guide](guides/judging.md) is the repository source for + `eval-magic docs judging`. diff --git a/docs/guides/byoh.md b/docs/guides/byoh.md index 738052e..32d9627 100644 --- a/docs/guides/byoh.md +++ b/docs/guides/byoh.md @@ -21,14 +21,14 @@ eval-magic run --harness cool-custom-harness `harness init` writes two files: - `.eval-magic/harnesses/cool-custom-harness.toml` is a commented descriptor with only `label` - enabled. + enabled, ready for you to declare its runner contract. - `.eval-magic/harnesses/cool-custom-harness-notes.md` records the source and harness version for every value you enable. -The label-only descriptor is usable. It falls back to `--no-stage`, inlines each `SKILL.md`, uses -`llm_judge` and runner-owned assertions for grading, and audits writes after dispatch. Read the -warnings from `run`; each warning names the lower-fidelity fallback carrying an undeclared -capability. +The label-only scaffold is not runner-ready until it declares `[dispatch]`, `[transcript]`, and +`[tools]`. Once those are valid, optional omissions have narrower fallbacks: no `skills_dir` forces +`--no-stage` and inlines each `SKILL.md`, while no built-in guard leaves the post-dispatch +`detect-stray-writes` audit as the filesystem safety net. `run` names those tradeoffs. For a descriptor that should not live in the project, pass it directly: @@ -36,12 +36,17 @@ For a descriptor that should not live in the project, pass it directly: eval-magic run --harness-file ./cool-custom-harness.toml ``` -Its `label` becomes the default harness for that invocation. +Its `label` becomes the default harness for that invocation. Every follow-up command `run` +generates — the printed Next: steps and each `eval-magic …` line in RUNBOOK.md — re-emits +`--harness-file`, because the descriptor can decide whether a comparison is valid (dispatch +templates, shadow isolation). Follow the generated commands verbatim: dropping the flag resolves a +different descriptor, and `dispatch`/`ingest` compare the resolved descriptor against the digest +`run` records in `conditions.json` and warn when the two differ. -## Add a dispatch recipe first +## Add a dispatch command first -The highest-leverage field is `[dispatch].exec_template`. It lets the generated `RUNBOOK.md` and -`dispatch-manifest.md` carry a copy-pasteable command for every task. +The highest-leverage field is `[dispatch].exec_template`. It is the command `eval-magic dispatch` +spawns for every task, so without it there is nothing for the runner to run. ```toml label = "cool-custom-harness" @@ -50,15 +55,44 @@ label = "cool-custom-harness" exec_template = ''' cool-cli run --cd {model_arg} \ "Read the file at and follow its instructions exactly." \ - > /final-message.md''' + > /cool-events.jsonl''' + +[tools] +write = ["file_change"] +patch = [] +shell = ["command_execution"] +read = ["file_read"] + +[transcript] +events_filename = "cool-events.jsonl" +surfaces_skill_invocation = false + +[transcript.extract.final_text] +where = { type = "agent_message" } +field = "text" ``` -The command has two requirements: +The runner-ready descriptor has two requirements: 1. Run the agent from the supplied ``. Each condition and repetition owns a private task repository there. -2. Recover the final reply at `/final-message.md`. Redirect stdout or copy the native - output there when the CLI cannot write the file itself. +2. Capture the native event stream at `/` and declare a + transcript reader that normalizes a non-empty final response. Use `[transcript.extract]` for a + flat JSONL stream or a named parser for a supported non-flat shape. + +Every runner-ready descriptor gets dispatch duration without another field. The runner measures +monotonic time inside the harness subprocess and stores it in `conversation.json`; resumed rounds +are summed. `ingest` carries that value into `timing.json` as `duration_source: "runner"`, while +tokens retain their transcript provenance. Native transcript duration is only a fallback for +historical or externally produced completion artifacts that have no runner measurement. + +The `[tools]` table is small but load-bearing beyond the descriptor. Grouping this harness's tool +names under `write`, `patch`, `shell`, and `read` is what lets the stray-write audit classify its +invocations, and what makes a frozen `transcript_check` tool pattern grade the same here as on +every other harness: names sharing a role are portable spellings of one another, so +`shell = ["cool_exec"]` is the whole opt-in. Spell only this harness's own names — cross-listing +another harness's is neither needed nor wanted. A tool left out of every role matches by its +native name alone. See `eval-magic docs judging`. Prove both requirements before a real eval: @@ -67,9 +101,9 @@ eval-magic harness lint .eval-magic/harnesses/cool-custom-harness.toml --probe ``` The probe renders the real command, asks for confirmation, invokes the harness CLI in a temporary -directory, and checks that the final-message file is nonempty. It can spend tokens and use network -services. Static lint runs first, and non-interactive use defaults to no; `--yes` explicitly accepts -the dispatch and `--probe-timeout SECONDS` bounds it. +directory, parses the configured event capture, and checks that the normalized final response is +nonempty. It can spend tokens and use network services. Static lint runs first, and non-interactive +use defaults to no; `--yes` explicitly accepts the dispatch and `--probe-timeout SECONDS` bounds it. ## Use the generated field reference @@ -93,6 +127,26 @@ The scaffold and resolved descriptor output are the installed references. Reposi can trace the underlying schema and adapter contracts from `docs/developer_overview.md` in a source checkout. +When a harness discovers project skills from more than its native staging directory, declare the +extra roots beside `skills_dir`: + +```toml +skills_dir = ".cool/skills" +additional_project_skill_dirs = [".claude/skills", ".agents/skills"] +config_dirs = [".cool", ".claude", ".agents"] +``` + +`skills_dir` is the only staging destination. The additional roots participate in sourced-codebase +shadow detection and `codebase.exclude_skill_sources`; eval-magic never stages into them. Every +path must be normalized, `/`-separated, and relative to the task repository. Its first segment must +also appear in `config_dirs`, keeping discovery, sibling filtering, and task-repository baselining +on one descriptor surface. + +`skills_dir` and the file your `[guard]` section stages (`hooks_file`, or `plugin_file` for a +plugin engine) are also what `run` hides from a sourced codebase's own linters and formatters, by +writing them into the project's ignore files. Declaring them correctly is all a descriptor has to +do; see `eval-magic docs codebase`. + ## Layer descriptors by field Descriptors load in this order: @@ -135,9 +189,20 @@ Use this sequence: 3. Run a small eval through `run`, dispatch, `ingest`, and `finalize`. 4. Confirm that every declared enhancement was exercised by the smoke run. -Scripted `turns` require `[conversation].resume_exec_template` plus transcript extraction of ordered -assistant messages and the native session ID. There is no fresh-session fallback: `run` rejects the -case when the harness cannot preserve the conversation. +Multi-turn evals — scripted `turns` and `responder` alike — require +`[conversation].resume_exec_template` plus transcript extraction of ordered assistant messages and +the native session ID. There is no fresh-session fallback: `run` rejects the case when the harness +cannot preserve the conversation. The responder itself needs nothing further from a descriptor; it +reads the agent's message out of the transcript and consults its own model through the dispatch +template you already declared, so it works on any harness that can resume. See +`eval-magic docs conversations`. + +A harness with a native read-only planning mode declares `[plan_mode]`: `plan_args` and +`act_args` fill a `{mode_args}` slot in both dispatch templates, and `[plan_mode.plan_file]` names +the file the harness writes its plan to, when it writes one. Verify each value the way you verify a +resume: one headless turn dispatched with the planning arguments must refuse an edit, and the same +session resumed with the act arguments must make it. `eval-magic harness show claude-code` is a +worked descriptor; the eval side is in `eval-magic docs conversations`. When a shadow preflight reports a live copy, isolate every initial and resumed eval-agent dispatch before setting `isolates_live_sources = true`. The per-harness remedies and verification procedure diff --git a/docs/guides/codebase.md b/docs/guides/codebase.md new file mode 100644 index 0000000..e59ff5f --- /dev/null +++ b/docs/guides/codebase.md @@ -0,0 +1,377 @@ +# Sourcing a codebase into a task environment + +> **Audience:** Eval authors choosing, pinning, and verifying a project for coding tasks. + +Every eval runs against a project codebase. Declare a `codebase` in `evals.json` and every +`(eval, condition, run)` environment is built from a checkout of it — with history, on a branch, +ready for the agent under test to work in. + +This matters for anything you cannot judge from a toy problem. Whether a skill makes an agent's +code *better* is not answerable when the task is small enough that any model succeeds. + +## Choose a source during `init` + +With no codebase option, `eval-magic init` uses the Weeknight example project at its pinned +baseline: + +```sh +eval-magic init +``` + +The generated `evals/evals.json` contains this `codebase` value: + +```json +{ + "url": "https://github.com/slowdini/eval-magic-fixture", + "ref": "b6d269c1cdedf7cadb53bacc41acaf5f2cdbe03f" +} +``` + +Choose another Git source by providing its URL and ref together: + +```sh +eval-magic init \ + --codebase-url https://github.com/slowdini/eval-magic-fixture \ + --codebase-ref b6d269c1cdedf7cadb53bacc41acaf5f2cdbe03f +``` + +`init` records those values without contacting the remote. `run` resolves the source and fails +before provisioning environments if the repository or ref is unavailable. + +For local work, name a directory already on disk or use the invocation directory itself: + +```sh +eval-magic init --codebase-path . +eval-magic init --codebase-cwd +``` + +A relative `--codebase-path` resolves from the directory where you invoke `init`. Relative path +inputs and `--codebase-cwd` are written relative to the generated `evals/` directory. An absolute +`--codebase-path` remains absolute. Local sources are convenient for iteration but carry the +portability limits described under "A `path` source is not reproducible elsewhere." + +The URL/ref, local path, and current-directory modes are mutually exclusive. The chosen source is +written into the eval file, so the committed configuration records which project the suite uses. + +## Choose the project scale + +### Start with Weeknight + +[Weeknight](https://github.com/slowdini/eval-magic-fixture) is a React and TypeScript meal-planning +web app. It has multiple routes, browser persistence, ingredient aggregation, tests, linting, and a +production build without a backend or external service. Use it for a first full-codebase eval or for +tasks where a compact project makes the agent's decisions easy to inspect. + +Suitable tasks include changing planner validation, extending the recipe filters, migrating stored +state, fixing shopping-list aggregation, or improving an interaction with focused tests. Pin the +project commit in the eval file even when a later revision exists; change the ref as a +deliberate eval-suite revision. + +### Use eval-magic as a complex project + +Use [eval-magic](https://github.com/slowdini/eval-magic) when the skill needs a larger codebase with +cross-module Rust behavior, schemas, generated artifacts, integration tests, and repository-level +contributor instructions. This project is appropriate when navigating and preserving those +contracts is part of what the eval should measure. + +This command scaffolds a pinned eval-magic source: + +```sh +eval-magic init \ + --codebase-url https://github.com/slowdini/eval-magic \ + --codebase-ref e30a5091c844e07f3aa664413aa7735a11b0a52a +``` + +The larger repository increases preparation, dispatch, and review work. Prefer Weeknight unless the +task genuinely needs the extra architectural surface. Project instructions and project-local skill +sources remain part of either project unless the eval opts out as described below. + +## Declare one + +A git repository, with an explicit ref: + +```json +{ + "skill_name": "working-with-tdd", + "codebase": { "url": "https://github.com/slowdini/example-project", "ref": "v1.4.0" }, + "evals": [ + { "id": "add-a-feature", "prompt": "...", "expected_output": "..." } + ] +} +``` + +Or a directory on this machine: + +```json +{ "codebase": { "path": "../../projects/legacy-service" } } +``` + +A relative `path` resolves against the directory holding `evals.json`, so a committed config means +the same thing in every clone of the skill. Unlike `files_root`, it may be absolute or point +outside the skill tree — that is the point of it. + +The config-level `codebase` is a default. Any eval can override it: + +```json +{ + "codebase": { "url": "https://github.com/slowdini/example-project", "ref": "main" }, + "evals": [ + { "id": "small-fix", "prompt": "...", "expected_output": "..." }, + { "id": "big-refactor", "prompt": "...", "expected_output": "...", + "codebase": { "path": "/srv/projects/monolith" } } + ] +} +``` + +## `ref` is required + +A git source must name a branch, tag, or full commit SHA. The runner resolves it and records the +commit, so a report says which tree it measured. An eval tracking whatever `main` happened to be +could not be re-run against the state it reported on, which is the point of recording provenance at +all. + +Resolution happens before any environment is created. An unreachable repository or a ref that does +not exist fails the run while it has still built nothing. + +## Project config and skill sources + +The sourced tree is preserved by default, including harness instructions, settings, plugins, and +project-local skills. For example, `CLAUDE.md`, `AGENTS.md`, `.claude/settings.json`, and +`.opencode/settings.json` remain visible in every comparison arm. + +Preserving project skills can contaminate a comparison when the codebase provides the +subject or one of its staged siblings. `run` records those matches in `plugin-shadow.json` with +`class: "codebase-sourced"`, separately from `class: "operator-environment"` findings caused by +global skills or installed plugins. Subject collisions are comparison-invalid; sibling collisions +follow the symmetric/asymmetric rules in `eval-magic docs isolation`. + +Opt an eval out of only the harness-discoverable project skill roots when the codebase's skills are +not part of the task being measured: + +```json +{ + "codebase": { + "url": "https://github.com/slowdini/example-project", + "ref": "v1.4.0", + "exclude_skill_sources": true + } +} +``` + +The default is `false`. When set to `true`, eval-magic moves every project skill root declared by +the selected harness out of each task environment before staging. It applies equally to both arms, +every repetition, revision mode, and `--no-stage`. Root instruction files and other harness config +remain in place. OpenCode, for example, excludes `.opencode/skills`, `.claude/skills`, and +`.agents/skills` because its descriptor declares all three discovery roots. For a BYOH descriptor +with no project skill roots, the setting is recorded and makes no filesystem change. + +Generated staging slugs are collision-safe: if the codebase owns that exact directory, the +runner backs it up, stages the evaluated copy for that arm, and restores the original during +cleanup. An explicit `--stage-name` remains stricter and refuses to clobber an occupied directory. + +The effective `exclude_skill_sources` value is recorded with each codebase in `conditions.json`, +every task in `dispatch.json`, every `run.json`, `benchmark.json`, and promoted `BASELINE.md`. + +## Framework files stay out of the project's own tooling + +Staged skills live *inside* the task repository, under the harness's skills dir. A project whose +lint or format step globs the whole tree would otherwise report eval-magic's own artifacts as +project failures — and only in the arm that stages a skill, because the control arm has no staged +skills to find. That biases every `command_check` running the project's checks, and it hands the +agent under test a red check listing files it did not create and must not touch. + +So `run` writes a delimited block into the project's own ignore files, naming the paths the runner +placed: + +``` +# >>> eval-magic framework files >>> +# Staged by `eval-magic run` so this project's own tooling does not report them. +# See `eval-magic docs codebase`. +/.eval-magic-outputs/ +/.claude/skills/ +/.claude/settings.local.json +# <<< eval-magic framework files <<< +``` + +The paths come from the selected harness descriptor — its `skills_dir` and the file its write guard +stages — plus the framework outputs directory, so a BYOH harness gets its own paths without +configuring anything. The block is written into **every** environment: both arms, every repetition, +revision mode, `--no-stage`, and `--dry-run`. An entry present in one arm only would trade one +asymmetry for another. + +Which ignore files get the block is detected from the codebase's own tooling: + +| Detected | Ignore file | Created when the project has none | +| --- | --- | --- | +| Prettier | `.prettierignore` | yes | +| ESLint | `.eslintignore` | no — ESLint 9's flat config no longer reads it | +| Stylelint | `.stylelintignore` | yes | +| markdownlint | `.markdownlintignore` | yes | +| Docker | `.dockerignore` | yes | + +Detection reads config-file markers and `package.json` dependencies anywhere in the tree. The list +is short because the ignore-file convention is: Python, Rust, and Go formatters have no ignore file +and already skip dot-directories, so they never see the staged skills in the first place. + +Name the ignore files yourself when the project's are somewhere detection will not look, or when +you want none at all: + +```json +{ + "codebase": { + "url": "https://github.com/slowdini/example-project", + "ref": "v1.4.0", + "ignore_files": ["tooling/.prettierignore"] + } +} +``` + +A declared list replaces detection rather than extending it, and each path is created if missing. +Paths are relative to the environment root and may not escape it. `"ignore_files": []` opts out +entirely, leaving every ignore file exactly as the codebase wrote it. + +`.gitignore` is never a target. The baseline commit force-adds harness config dirs, so an entry +there would hide nothing from Git — but it *would* hide the staged skills from every +`.gitignore`-aware tool the agent uses, such as `rg`, which would damage the treatment arm instead +of protecting it. + +Unlike `exclude_skill_sources`, no artifact records this: the written file is committed into each +environment's baseline, so `git show refs/eval-magic/baseline:.prettierignore` inside any task +environment is the evidence, per arm and per run. `run` also prints the files it wrote. + +## What the environment contains + +Each dispatch gets its own private environment holding: + +- the codebase, checked out at the resolved commit, with its history intact +- no remotes — nothing in the environment can reach or push to the source it came from +- hooks disabled, and a fixed committer identity for the runner's own commit +- the branch the codebase itself was on: the branch a `ref` names, or the repository's default + branch when the ref is a tag or a SHA +- `refs/eval-magic/baseline`, marking the state the agent started from + +Every selected eval must resolve an effective codebase, either from the top-level default or an +eval-level override. Validation rejects a configuration that supplies neither. + +## The baseline ref is what the run is measured against + +Nothing writes into an environment after that ref is written, so it names exactly what the agent +started from — and everything the agent did is the difference from it. + +During `ingest`, Git measures that difference. Each run gets: + +- `diff-scope.json` — `files_touched`, `lines_added`, `lines_removed`, and `hunks`, plus the list of + changed files with a status of `added`, `modified`, or `deleted` +- `diff.patch` — the diff itself, which is the evidence a judge reads to answer whether the work was + any good. It always exists; for a run that changed nothing it is empty. A diff past the capture + cap is cut at a line boundary and carries a marker saying so, and `patch.truncated` in + `diff-scope.json` records it. +- `judge-evidence.md` — the bounded grading input that combines this diff with the task, completion + state, conversation, and tool summary. See `eval-magic docs judging` for its limits, trust + boundary, and retained-baseline behavior. + +What counts is what Git counts, under the same rules the baseline commit was built under: + +- The codebase's own `.gitignore` holds, so a run that compiles does not report its build output as + thousands of touched files. +- Overlay files and staged skills count even when the codebase ignores their paths — they are + committed into the baseline regardless, so a change to one is always visible. +- Framework artifacts under `.eval-magic-outputs/` never count. +- The task-local scratch directory never counts. Dispatch prompts designate `/tmp/` for + temporary work and tell the agent to use it, so what lands there is the framework's instruction + being followed, not the change under measurement. It is excluded from the project's own ignore + files for the same reason, and it never reaches `diff.patch`, so a judge never reads scratch notes + as the deliverable. +- A nested repository's internals never count: Git tracks no path with a `.git` component. +- A rename counts as two touched files, one created and one deleted. +- A binary file counts as one touched file, contributing no lines. + +A `diff_scope` assertion gates `max_files_touched`, `max_lines_changed` (added plus removed), or +both, against exactly these numbers. + +## One checkout per iteration + +Every environment a run provisions — each `(eval, condition, run)` cell — is built from one cached +checkout per distinct codebase, materialized once per iteration under `iteration-N/.codebase/` +when the run prepares. +Environments are provisioned from that cache with `git clone --local`: Git hard-links the object +store instead of copying it and checks out a fresh working tree, so `--runs 10` against a large +repository costs one clone plus a working tree per environment, not a full copy of the tree and +its history per environment. + +Shared objects are content-addressed and immutable — Git never rewrites an object once written — +so each environment is still an independent working tree with an independent history. Commits, +branches, and edits in one environment are invisible to the others and to the cache. + +Where hard-linking is unavailable (the cache and the environments on different filesystems) or the +source carries no Git history to clone, environments fall back to a plain copy of the cache. The +result is the same tree, provisioned more slowly. + +## `files` is an overlay + +`files` and `files_root` still work, and are applied *on top* of the codebase at their declared +paths. Seeding a task-specific file into a real project is the common case: + +```json +{ + "id": "add-a-feature", + "prompt": "Implement what docs/TASK.md describes.", + "expected_output": "the feature, with tests", + "files": ["docs/TASK.md"] +} +``` + +An overlay file replaces a codebase file at the same path. + +The baseline the runner commits respects the codebase's `.gitignore`, so ignored build output stays +out of it. Overlay files and staged skills are committed regardless of what the codebase ignores — +which is also what keeps them inside every later measurement. + +## A `path` source is not reproducible elsewhere + +Someone reading your published results cannot resolve `../../projects/legacy-service`. Their machine +has that directory somewhere else, or not at all. Nothing can fix that, so the artifacts label it: +the record carries `host_local: true`, the run prints a warning, and the `BASELINE.md` row says so. + +Where the directory is itself a Git repository, its `origin` URL and the resolved commit are +recorded too, and *those* resolve anywhere. Prefer a `url` source for anything you intend to +publish. + +A `path` source is materialized as a clean checkout of its committed state. Uncommitted work in the +source directory is not carried into the environment; the run warns when the source is dirty. + +## Verify the result + +From a prepared iteration directory, inspect one environment: + +```sh +cd env-g1-with_skill +git log --oneline | head +git remote -v +git rev-parse refs/eval-magic/baseline HEAD +git status --porcelain +``` + +`git remote -v` and `git status --porcelain` are both empty, and the two revisions match: the +baseline ref names exactly what the agent started from. + +After a dispatch and `ingest`, read what the run produced: + +```sh +jq '{files_touched, lines_added, lines_removed, hunks, files, patch}' diff-scope.json +head -50 diff.patch +sed -n '1,240p' judge-evidence.md +``` + +The same difference, spelled by Git itself, is `git diff refs/eval-magic/baseline` inside the +environment. + +The resolved commit and effective skill-source policy appear in `conditions.json`, each `run.json`, +`benchmark.json`, and the `BASELINE.md` written by `promote-baseline` — alongside the skill the run +measured, which is recorded the same way: + +```sh +jq '.codebases, .skill_source' conditions.json +``` + +See `eval-magic docs isolation` for what the skill side of that record means. diff --git a/docs/guides/conversations.md b/docs/guides/conversations.md new file mode 100644 index 0000000..1cc9d64 --- /dev/null +++ b/docs/guides/conversations.md @@ -0,0 +1,253 @@ +# Multi-turn conversations + +Most evals are one shot: the agent gets a prompt, works, and answers. Some tasks +are not like that. A realistic request often needs a decision from the user part +way through, and an eval that cannot supply one measures an agent talking to a +wall. + +An eval declares one of two ways to supply those answers. They are alternatives, +not layers — declaring both is a configuration error. + +- **`turns`** — an authored script. You say exactly what the user says, and in + what order. Use it when the exchange is the thing under test and you want it + identical in every run. +- **`responder`** — a policy that derives each answer from what the agent just + said. Use it when you do not know what the agent will ask, which is the normal + case for a real task against a real codebase. + +Declaring neither leaves the eval one shot. + +Both need a harness that can resume its own session, so a follow-up reaches the +agent that asked rather than a fresh one. `eval-magic run` rejects the eval up +front when the selected harness cannot. `eval-magic harness list` names the +`conversation-resume` capability for every harness that has it. An eval that +begins in the harness's plan mode needs the `plan-mode` capability as well; see +"Starting in plan mode" below. + +## The responder + +```json +{ + "id": "add-request-caching", + "prompt": "Requests to the pricing API are slow. Can you add caching?", + "expected_output": "A working cache with the pricing endpoint under 100ms.", + "responder": { "type": "llm", "max_turns": 8 } +} +``` + +- **`type`** is required. `llm` is the only responder: a small model, consulted + once after every round through the same harness as the agent under test. + Choose it with `eval-magic run --responder-model`; omit that flag and the + consultation runs on the harness's default model. +- **`max_turns`** bounds how many follow-ups the responder may synthesize. The + opening prompt is not one of them. It defaults to 8. + +Every turn the responder produces is recorded in the run's `conversation.json` +with an `origin` naming the responder and, when it offered one, its one-line +reason for answering that way. A turn the eval authored — the opening prompt, a +scripted turn — carries no `origin`; the one turn the runner itself authors, a +plan-mode eval's approval, carries `origin.runner` instead. + +The full prompt and verdict of every consultation are kept on disk under the +run's `responder/turn-/`, so you can audit what the responder was shown and +what it wrote without rerunning anything. + +## What the responder is shown + +**Only what the agent already knows.** Each consultation carries the eval's +opening `prompt`, every reply the responder has already given, and the agent's +last message. It does not carry `expected_output` and it does not carry the +assertions: those are the grading criteria, and a responder that had read them +could hand the agent the rubric. + +It is told to answer as the person who asked for the work — take whatever the +agent marked as recommended, else the simplest option and the least work; add no +requirements; invent no facts; write no code; keep it short. + +It answers with one of three verdicts: + +| Verdict | What it means | +| --- | --- | +| `answer` | What the user says next. Delivered as the following turn. | +| `done` | The agent is reporting the task finished and waiting on nothing. | +| `cannot_answer` | It could not answer without inventing something. | + +Because the responder decides `done`, completion is a judgement rather than the +absence of a question mark — and the judgement is recorded with its reason, so +a run that stopped early is legible rather than mysterious. + +## What is never delivered + +A reply that fails any of these checks is not sent to the agent. The run stops +instead, because an undelivered reply is a loud, greppable stop, while a bad one +enters the transcript as an ordinary user turn and is graded as though the +exchange really happened. + +| Rejected when the reply | Recorded cause | +| --- | --- | +| is blank | `empty_reply` | +| runs past 2000 bytes | `reply_too_long` | +| contains a fenced code block | `reply_contains_code` | +| repeats the previous reply verbatim | `reply_repeated` | + +The length and code rules are the same rule twice: a simulated user answers in +sentences, so anything longer means the responder started doing the agent's work, +and crediting the agent under test with work it did not do would corrupt the +result. A repeat means the exchange is circling, and spending the remaining +turns on it would only reach the same place more expensively. + +A consultation that never produces a reply stops the run the same way, with its +own cause: `declined` when the responder honestly refused, and +`dispatch_failed`, `dispatch_timed_out`, `missing_verdict`, or +`malformed_verdict` when something broke. One outcome, because the run ended +mid-task either way; separate causes, because an honest refusal and a broken +dispatch call for different fixes. + +## How a conversation ends + +| Recorded as | When | +| --- | --- | +| `completed` | The responder judged the agent finished. The run stops rather than burning its remaining turns. | +| `stopped`, `responder_cannot_answer` | The responder produced no usable reply. `responder_outcome.cause` says why. | +| `stopped`, `max_turns_reached` | The agent was still asking at the bound. | +| `stopped`, `plan_not_presented` | A plan-mode session ended its planning phase with no plan to approve. | +| `timed_out` | The task outran `dispatch --timeout`. | + +A `stopped` conversation is recorded, not failed: `dispatch` exits zero and +`ingest` still records the run. But both responder stops end the conversation +with the task unfinished, so `dispatch` warns about each one by name and +`aggregate` counts them per condition in `benchmark.json`'s +`validity_warnings`. That count is the one to read first: one arm truncated more +often than the other is a threat to the comparison, not just to the run. + +## Dispatch timing + +Every task driven by `eval-magic dispatch` records `duration_ms` in its completed, stopped, or +timed-out `conversation.json` file. This is a runner-owned monotonic measurement of time spent +inside the eval-agent harness subprocess. For a multi-turn task, it is the sum of the initial +dispatch and every resumed round. + +The measurement excludes responder consultations, judge dispatches, time waiting in the task +queue, and runner bookkeeping. A timeout retains the elapsed time spent waiting for the killed +harness process, while a harness command that fails writes no completion artifact. + +During `ingest`, `timing.json` becomes the canonical location for the measurement. New timing files +record provenance per metric: transcript-normalized tokens use `token_source: "transcript"`, and +runner duration uses `duration_source: "runner"`. `run.json` does not duplicate the runner timing. +Historical or externally produced completion artifacts without `duration_ms` fall back to a native +transcript duration when the harness exposes one; existing `timing.json` files are preserved unless +you pass `--overwrite`. + +## Cross-harness behaviour + +The responder needs no per-harness support and no descriptor field. It reads +`TranscriptSummary::final_text`, which every harness's parser already +normalizes, replies through the existing `{prompt_arg}` slot, and runs its own +consultations through the same `[dispatch].exec_template` a judge uses. Any +harness that can resume a session can run a responder eval. + +Nothing is read from a harness-native question tool, and nothing needs to be: a +dispatch runs headless with stdin detached, so a tool that asks the user has no +channel to be answered on. Free text is the only mechanism that fits, and it +happens to be the portable one. + +Consultations run in the run's own `responder/` directory, which sits above the +task environment. That is deliberate — a consultation must not be able to write +into the codebase under measurement, and must not pick up that codebase's +`CLAUDE.md` or `AGENTS.md` as instructions to itself. + +## Scripted turns + +```json +{ + "id": "clarify-before-editing", + "prompt": "The due date is wrong. Fix it.", + "expected_output": "Asks which timezone before editing.", + "turns": [ + { + "prompt": "The affected users are all in US timezones.", + "deliver_when": "agent_asks", + "agent_response_matches": "(?i)time ?zone" + }, + { "prompt": "It is a date-only field.", "deliver_when": "always" } + ] +} +``` + +Each turn is delivered in order. `deliver_when: always` delivers +unconditionally; `agent_asks` delivers only when the preceding response contains +a question mark, and `agent_response_matches` adds a regex the response must +also match. A turn whose gate is unmet stops the conversation and is recorded as +`agent_did_not_ask` or `agent_response_mismatch` — a real result about the +agent, which is usually the point of scripting the exchange. + +## Starting in plan mode + +A real session often begins in the harness's plan mode: the agent reads and +plans but cannot edit, presents a plan, and implements only once the person +approves it. An eval declares that shape with `plan_mode`: + +```json +{ + "id": "add-request-caching", + "prompt": "Requests to the pricing API are slow. Can you add caching?", + "expected_output": "A working cache with the pricing endpoint under 100ms.", + "plan_mode": true, + "responder": { "type": "llm" } +} +``` + +The session runs in two phases, in one native session: + +1. **Planning.** The opening prompt is dispatched in the harness's native plan + mode, so the agent explores read-only. If it asks a question, the responder + answers it and the session stays in plan mode. +2. **Implementation.** Once the agent has presented its plan, the runner + approves it with one fixed message and resumes the same session in act + mode. The message is always `The plan is approved. Implement it now.` From + there the eval's `turns` or `responder` proceed exactly as they would + without plan mode. + +The approval is fixed rather than judged, so the transition is identical in +every run and both arms: what varies between conditions is the plan and the +implementation, never how the runner reacted to them. + +### How the runner knows the plan is ready + +Two signals, tried in this order: + +| Signal | When it applies | +| --- | --- | +| `plan_file` | The harness writes the plan it presents to a file, and its descriptor declares where (`[plan_mode.plan_file]`). A planning round that wrote one has presented its plan, and the file's content is the plan. Claude Code writes to `~/.claude/plans`. | +| `responder` | The eval declares a `responder`, which is told the agent is planning. Its `done` verdict means the plan is ready, and the agent's last message is the plan. | + +A harness that writes no plan file needs the responder to decide, so `run` +rejects a plan-mode eval there unless it declares one. With a plan file the +responder is optional; if the agent never writes one and there is no responder +to ask, the run stops with `plan_not_presented`. + +`eval-magic harness list` shows `plan-mode` for every harness that can start a +session in plan mode. `run` rejects a plan-mode eval on one that cannot, before +any environment is built. + +### What is recorded + +- `outputs/plan.md` holds the approved plan, and the judge evidence bundle + renders it in a section of its own. +- Every user message in `conversation.json` carries `mode` (`plan` or `act`), + and the approval turn carries `origin.runner: plan_approval`. +- `conversation.json`'s `plan` names the round the plan was presented in, the + round the approval opened, and which signal fired. +- A responder's `max_turns` is one budget across both phases: planning-phase + answers count toward it. +- A plan-mode run whose session never left the planning phase — whatever + stopped it — is counted per condition in `benchmark.json`'s + `validity_warnings`, because it never attempted the task. +- An agent that tries to edit while planning is refused by the mode itself. + That refusal is recorded in `permission-denials.json` as behavioral evidence + and marked `plan_mode_attributed`, but it raises no validity warning: the + mode refusing a write is the mode working. + +Where the harness writes its plan file, the write guard and the stray-write +audit allow that root beside the task environment. The plan file lands where +the harness puts it in any session; `plan.md` is the copy the judge reads. diff --git a/docs/guides/guard.md b/docs/guides/guard.md new file mode 100644 index 0000000..92cc25b --- /dev/null +++ b/docs/guides/guard.md @@ -0,0 +1,165 @@ +# Configuring guarded commands + +> **Audience:** eval authors deciding which development commands an agent may run inside a task +> environment. + +The write guard combines a fixed containment boundary with an eval-authored command policy. Use +the `guard` field in the `evals.json` file to grant the tools or command prefixes a task needs. +Without an explicit `guard` field, eval-magic detects packaged profiles from the staged task tree. + +## Understand the two policy layers + +Containment checks run before command allowances. A command policy cannot override these checks. +A command that violates both layers is denied once, with a verdict that names each blocking reason, +so the denial never points at a fix that cannot unblock the command: + +- Direct write and patch tools must target the task environment. +- Shell redirects and `tee` targets must resolve inside the task environment. +- Remote Git mutations and repository-routing escapes are blocked. +- Recognized package, build, and edit destinations must stay inside the task environment. Global + and user installation modes are blocked unless the classifier can prove an in-task destination. +- Recognized filesystem mutations must stay inside the task environment. The classifier reads + `touch`, `mkdir`, `rm`, `cp`, `mv`, and `install`, resolving their operands and their + `-t`/`--target-directory` destinations against the invocation directory; because `mv` unlinks + what it moves, its sources are checked too. A glob or brace confined to one path component under + an in-task directory is allowed, since it can only expand to entries of that directory; a + variable or command expansion names no path and is blocked. + +These families are recognized for containment only. Creating a directory or copying a file inside +the task environment needs no allowance, so `guard` configuration is unchanged by them. + +After those checks, the command policy handles recognized development mutations. A recognized +command that has no matching allowance is blocked. A tool claimed by `allow_commands` also has its +other subcommands blocked. Commands that neither the containment classifier nor the command policy +recognizes remain best-effort allowed and are inspected by `detect-stray-writes` after the run. + +## Choose the configuration scope + +A config-level `guard` field is the default for every eval: + +```json +{ + "skill_name": "rust-maintainer", + "guard": { + "allow_tools": ["cargo"] + }, + "evals": [ + { + "id": "repair-workspace", + "prompt": "Repair the workspace and run its tests.", + "expected_output": "The workspace tests pass." + } + ] +} +``` + +A per-eval `guard` field completely replaces the config-level field. It does not merge with it. +Even an empty object disables the config default and automatic profile detection for that eval: + +```json +{ + "skill_name": "web-maintainer", + "guard": { + "profiles": ["language/javascript"] + }, + "evals": [ + { + "id": "serve-next-app", + "prompt": "Start the development server.", + "expected_output": "The server starts.", + "guard": { + "allow_commands": ["npm run dev"] + } + } + ] +} +``` + +Any explicit `guard` field disables automatic detection. When it names `profiles`, only those +profiles are expanded. + +## Choose allowance granularity + +The `guard` object accepts three arrays: + +- `allow_tools` contains executable basenames. An entry such as `cargo` permits all Cargo + subcommands after containment checks. +- `allow_commands` contains one literal shell command per entry. Each entry is a token prefix, so + `cargo test` permits `cargo test --workspace` but not `cargo build`. +- `profiles` contains packaged profile IDs. Profile commands are added to `allow_commands`. + +Command rules cannot contain pipes, separators, redirects, variable expansions, or command +substitutions. Eval validation rejects those shapes. Matching normalizes an executable path to its +basename, skips leading environment assignments, and understands the `env`, `command`, `exec`, +`nice`, and `timeout` wrappers. A literal command passed through `sh -c`, `bash -c`, or `zsh -c` is +matched recursively. Every segment of a compound command must be allowed independently. + +For example, this policy permits the listed Next.js lifecycle scripts but claims no other npm +subcommands: + +```json +{ + "guard": { + "allow_commands": [ + "npm run dev", + "npm run build", + "npm run start" + ] + } +} +``` + +With that policy, `npm run dev -- --hostname 127.0.0.1` is allowed and `npm install` is blocked. +Use `allow_tools` only when every subcommand of the tool is appropriate for the eval. + +## Use packaged profiles + +Packaged profiles provide lightweight defaults. Detection is recursive, so a frontend and backend +in the same task environment can activate multiple profiles. The detector skips `.git`, +`.eval-magic-outputs`, harness configuration and staged-skill directories, `target`, +`node_modules`, and `.venv`. + +The packaged profiles are: + +- `language/rust` is detected from `Cargo.toml`. It allows `cargo build`, `check`, `test`, `run`, + `fmt`, and `clippy`. +- `language/javascript` is detected from `package.json`. It allows npm install, CI, test, build, + dev, start, lint, and typecheck commands, plus corresponding pnpm, Yarn, and Bun install, add, + test, build, dev, start, lint, and typecheck commands. +- `framework/nextjs` is detected when `package.json` declares a `next` dependency. It allows npm, + pnpm, Yarn, and Bun dev, build, and start scripts, plus direct `next` invocations through `npx`, + `pnpm exec`, `yarn`, and `bunx`. +- `language/python` is detected from `pyproject.toml`, `setup.py`, or a + `requirements*.txt` file. It allows pip installs, Python module invocations for pip, build, + pytest, and unittest, and direct `pytest`. + +Name profiles explicitly when the files in the task tree are not the policy you want: + +```json +{ + "guard": { + "profiles": ["language/javascript", "framework/nextjs"], + "allow_commands": ["npm run integration"] + } +} +``` + +Explicit commands and expanded profile commands are deduplicated in the effective policy. + +## Audit the effective policy + +The run preflight resolves harness support, staging, `--guard`, and `--no-guard` into one effective +state. `run` records that state as `guard_armed` in `conditions.json`, mirrors it as `guard` in +`dispatch.json`, and carries it into `benchmark.json`. The `BASELINE.md` file written by +`promote-baseline` renders the value as `true` or `false`; a historical conditions file without the +field renders as `unknown`, not `false`. + +Each task in `dispatch.json` records its fully expanded `guard_policy`. The armed marker records the +same policy as `guardPolicy`, so the live hook and the campaign plan cannot resolve defaults +differently. `detect-stray-writes` reads the frozen task policy from `dispatch.json` and applies the +same classifier after the run. A legacy dispatch without `guard_policy` uses an empty command +policy rather than guessing which defaults applied. + +The command policy is not a complete shell sandbox. Keep task environments isolated, inspect guard +denials and stray-write findings during ingest, and use narrow `allow_commands` entries when the +eval does not need every operation a tool exposes. diff --git a/docs/guides/isolation.md b/docs/guides/isolation.md index 1299dbb..1f704ed 100644 --- a/docs/guides/isolation.md +++ b/docs/guides/isolation.md @@ -1,11 +1,29 @@ -# Isolating dispatches from live skill sources +# Dispatch isolation and sandbox boundaries -> **Audience:** operators whose run printed a skill-shadow warning or whose -> `plugin-shadow.json` or `benchmark.json` reports a live copy of an eval skill. +> **Audience:** operators preparing eval dispatches, troubleshooting sandbox failures, or +> resolving live skill-source warnings. -Use this guide after eval-magic reports a discoverable live skill. It explains how to exclude that -source from eval-agent dispatches and how to verify the result. Normal task-repository isolation and -the write guard are separate concerns documented by `eval-magic run --help`. +Several boundaries shape an eval dispatch. Use this guide to distinguish inherited +operator restrictions from task protections and skill-source isolation, then follow the relevant +harness section for troubleshooting or live-source remedies. + +## Understand the boundaries + +- **Operator sandbox**: the environment around the process launching `eval-magic dispatch`. + An operator Codex session can restrict the runner and the harness commands it spawns. +- **Harness task sandbox**: the harness's own operating-system restrictions on the eval agent. + The generated Codex task command uses `--sandbox workspace-write`. This inner sandbox cannot + grant access denied by the operator's outer sandbox. +- **Eval guard**: eval-magic's tool and command checks enforce task containment for recognized + operations and apply the eval-authored command policy. These checks complement the harness + sandbox; see `eval-magic docs guard` for their scope and limits. +- **Skill-source isolation**: excluding competing live skill copies from what an eval agent can + discover or load, so the treatment comparison measures the intended skills. A task sandbox or + eval guard does not by itself exclude these sources. + +Every task also has a private Git repository prepared by eval-magic; see `eval-magic run --help` +and `eval-magic docs codebase`. That repository boundary is independent of the permissions the +operator and harness sandboxes enforce. ## Why a live copy matters @@ -20,9 +38,14 @@ Sibling collisions have two outcomes: - A sibling visible in only one arm is comparison-invalid because its effect cannot be separated from the skill under test. -The preflight reports what the environment makes discoverable. Transcript evidence can later show -what a dispatch loaded. Eval-magic does not parse shell templates to infer that a flag or environment -variable isolates the process. +The preflight reports two source classes in schema-v3 `plugin-shadow.json`: + +- `operator-environment` — global skills, enabled plugins, and other sources inherited from the + machine running eval-magic. +- `codebase-sourced` — matching project-local skills preserved from the task codebase. + +Transcript evidence can later show what a dispatch loaded. Eval-magic does not parse shell +templates to infer that a flag or environment variable isolates the process. Apply the remedy to **every eval-agent command**, including every resumed turn of a scripted eval. Isolating only the first round allows the live copy to return on the next round. Judge commands do @@ -43,6 +66,34 @@ editing global configuration. Project-local staged skills still load under every ## Isolate Codex +### Diagnose nested sandbox failures + +If the same generated task command succeeds in an ordinary terminal with +equivalent inputs and configuration, but fails inside the operator Codex session with +`Operation not permitted`, the outer sandbox may be preventing the dispatch. The error +`Operation not permitted` alone does not establish a nested-sandbox problem. Compare the command, +working directory, inputs, and resolved harness configuration before choosing this remedy. + +Commands spawned from an operator Codex session inherit its sandbox boundaries. The inner task +sandbox cannot grant access denied by that outer process. The outer environment may also block +operations needed to create the inner sandbox. Adding a writable directory alone does not +guarantee that nested dispatch will work. + +Choose one of these remedies: + +1. **Preferred:** run the `eval-magic dispatch` command from the generated `RUNBOOK.md` in that + ordinary terminal, using the runbook's working directory and arguments. +2. **Alternative:** approve or escalate the outer launch of `eval-magic dispatch` where the + operator surface and policy support it. Limit the approval to the workspace and process access + required to launch the guarded tasks, including creation of their inner sandboxes. + +Keep the task command's `--sandbox workspace-write` and eval guard enabled under either remedy. +Do not replace the generated harness command with an unrestricted command or disable the guard. +Eval-magic does not detect or bypass the operator's sandbox; the remedy belongs to the outer +execution environment. + +### Exclude live skill sources + Choose the remedy that matches each reported source: | Remedy | Hides | Leaves visible | Caveat | @@ -80,9 +131,21 @@ label = "claude-code" isolates_live_sources = true ``` +The declaration covers only `operator-environment` findings. It does not claim that skills sourced +from the task codebase are isolated. Use `codebase.exclude_skill_sources: true` for that separate +policy when project skills should not participate; see `eval-magic docs codebase`. + The declaration does not disable detection. `plugin-shadow.json` retains every source and its -intrinsic severity as provenance. `run` presents the finding as informational, and `aggregate` -omits the warning only while no transcript evidence contradicts the declaration. +intrinsic severity as provenance. `run` presents operator-environment findings as informational, +and `aggregate` omits those warnings only while no transcript evidence contradicts the declaration. +Codebase-sourced findings remain warnings regardless of this descriptor setting. + +The declaration is written from the descriptor resolved at prep time, so the remedy must ride along +with every follow-up. When the descriptor arrives via `--harness-file`, the generated RUNBOOK.md +and Next: commands re-emit the flag — follow them verbatim. Dropping it silently reverts to the +un-overlaid descriptor while the declaration stands, the comparison-invalid state this guide exists +to prevent. `dispatch` and `ingest` compare the resolved descriptor against the digest `run` +records in `conditions.json` and warn when the two differ. Do not set it when: @@ -125,8 +188,8 @@ ID rather than the total list length because staged and bundled skills remain pr produce an init record under each `turn-N/`; inspect a resumed turn as well. Codex and OpenCode captures do not provide the equivalent roster used by eval-magic. Verify those -harnesses by checking every rendered eval-agent command in `RUNBOOK.md` and -`dispatch-manifest.md`, then use `isolates_live_sources` to record the operator assertion. +harnesses by checking the eval-agent command `dispatch-manifest.md` says the runner will spawn, +then use `isolates_live_sources` to record the operator assertion. ### `claude plugin list` does not prove isolation @@ -134,6 +197,85 @@ harnesses by checking every rendered eval-agent command in `RUNBOOK.md` and dispatch's setting-source selection. A plugin can appear there and remain absent from the dispatch, or the reverse. Use the dispatch's init event. +## Treatment skills are copies + +`skill_name` in the `evals/evals.json` file accepts either one skill name or an ordered, +non-empty list: + +```json +{ + "skill_name": ["review-workflow", "review-verification"], + "evals": [ + { + "id": "review-change", + "prompt": "Review this change.", + "expected_output": "A prioritized review." + } + ] +} +``` + +With a list, `--skill` selects the eval owner: the member whose `evals/` directory supplies the +definitions and overlay files, and whose name owns the workspace and promotion destination. The +owner must appear in the list. `--stage-name` is unavailable because one override cannot name +several staged skills. + +Every treatment member is copied into the eval home before any dispatch runs, and each condition +stages from those copies. Mode A stages all treatment members in `with_skill` and none in +`without_skill`. Other siblings from `--skill-dir` remain ambient in both arms. Mode B snapshots +and stages the complete set in both revisions. A scalar `skill_name` retains the existing +single-skill paths and artifacts. + +Each copy is the working tree as it sits on disk, not a checkout of a commit. Evaluating an +uncommitted revision is the ordinary case, and in a `--mode revision` run the edit under test is +uncommitted by definition. What the run measured is recorded rather than inferred in +`conditions.json`, each `run.json`, `benchmark.json`, and the `BASELINE.md` file written by +`promote-baseline`: + +```sh +jq '.skill_source' conditions.json +``` + +`dirty: true` means the recorded revision alone does not identify what ran. Commit the treatment +skills before a run whose result you intend to publish. + +The copy freezes the treatment, not the assertions. `grade` reads `assertions` and +`skill_should_trigger` from the live `evals/evals.json` and everything else from the copy, because +the judging loop authors assertions from the run's own evidence, after the dispatch they grade. +Each `grading.json` records which file supplied its assertions. See `eval-magic docs judging`. + +Ambient skills staged by `--skill-dir` are copied the same way, and the roster is captured once +when the run resolves. For a multi-skill treatment, `skill_source.eval_owner` names the owner and +`skill_source.skills` records every treatment member's resolved source and revision. The +`siblings` field, when present, names ambient skills staged in both arms. + +The eval home sits outside the skill's own repository: under `$XDG_DATA_HOME/eval-magic` +(or `~/.local/share/eval-magic`), in a directory named for the skill directory it serves. +`run` prints the path it chose, and every command it suggests carries `--workspace-dir`, +so there is nothing to remember. `EVAL_MAGIC_WORKSPACE_DIR` moves the default; +`--workspace-dir` overrides both. + +Copying does not remove the live directories from the machine, so a dispatch can still read one by +absolute path. `detect-stray-writes` checks every treatment source and reports that as a live-source +read. `aggregate` carries it into `validity_warnings` for the same reason a discoverable plugin +copy is carried there: the arm may not be comparing what it claims to. + +The check compares resolved paths, so a symlinked route to a live directory is reported like the +direct one. Aliases are the ordinary case rather than an evasion: on macOS `$TMPDIR` sits under +`/var`, itself a link to `/private/var`, so a dispatch that builds an absolute path from its own +environment spells one directory a different way than the runner recorded it. + +## The task repository is a separate boundary + +Skill-source isolation is about what a dispatch can *load*. The task repository is about what it can +*reach*: every dispatch runs in its own private environment, a Git repository with no remotes and +hooks disabled, marked with `refs/eval-magic/baseline` at the state the agent started from. That +codebase-backed boundary holds for every eval; task-specific `files` are overlays on the sourced +tree. See `eval-magic docs codebase`. + +The two are independent. An environment can be a faithfully isolated repository while the dispatch +still loads a live skill source, and a shadowed skill is not made safe by the repository boundary. + ## When a source cannot be isolated Do not declare isolation. Retain the validity warning as the record of a known threat. A symmetric diff --git a/docs/guides/judging.md b/docs/guides/judging.md new file mode 100644 index 0000000..7b07eb5 --- /dev/null +++ b/docs/guides/judging.md @@ -0,0 +1,284 @@ +# Judge evidence bundles + +> **Audience:** eval authors and operators deciding whether an LLM verdict has enough evidence to +> trust. + +`ingest` writes one `judge-evidence.md` beside every recorded run. This Markdown bundle is the +primary input for every LLM judge task for that run: eval-magic persists it once and inlines those +exact bytes into each judge prompt. Read the bundle when a verdict is surprising, when a truncation +marker appears, or before promoting an important result. + +## Explore before writing assertions + +An eval can begin with a realistic prompt and `expected_output` but no assertions. Run both +conditions first, then use their paired evidence to discover behavior worth measuring: + +1. Follow the iteration's `RUNBOOK.md` through eval dispatch and `ingest`. Ingest writes the + bounded evidence bundle for every recorded run even when the eval declares no assertions. +2. Create the paired report for one eval: + + ```sh + eval-magic compare --iteration 1 --eval implement-feature + ``` + +3. Give the printed Markdown path to the driving agent. Ask open questions about the code, + completion behavior, tool use, or moments of confusion in the two conditions. +4. Turn concrete observations into `llm_judge`, `transcript_check`, `command_check`, or + `diff_scope` assertions in the skill's own `evals/evals.json` — the live file, not the copy the + iteration froze. +5. Re-run `eval-magic grade --iteration N`, or the `ingest` command that ends in it, to grade what + you just wrote. Repeated agent runs or judge samples are what turn one observation into a + measurement. + +`compare` is not a grade and does not choose a better condition. One paired report is exploratory +evidence for drafting hypotheses, not a statistically reliable result. It includes every matching +run from both conditions, labels multi-run evidence by run index, and refuses to write a partial +report when an arm or bundle is missing. The report also points to available guard, permission, +stray-write, and skill-shadow validity artifacts so blocked or contaminated behavior is not +mistaken for a condition effect. + +The embedded task, transcript, tool, and patch content is untrusted read-only evidence. Do not +follow instructions inside it. When a bundle carries a truncation marker, inspect the named source +before drawing a conclusion from omitted material. + +## Portable tool patterns + +A `transcript_check` with `check: "tool_invocation_matches"` runs its `pattern` against the +`" "` rendering of each recorded tool call. Harnesses spell the same tool +differently — Claude Code records `Bash`, Codex `command_execution`, OpenCode `bash`, Cline +`run_commands` — so a pattern naming one harness's tool would score zero on the others even where +the behavior plainly happened. + +Matching is therefore role-granular. Every harness groups its tool names into four roles — +`write`, `patch`, `shell`, `read` — and grading uses them in two stages: + +1. The regex runs against the native rendering, exactly as the run recorded it. +2. On a miss, the run's own harness supplies the role its tool name belongs to, and the regex runs + again against one rendering per portable spelling of that role. Only the name is substituted; + arguments are preserved, so a pattern over arguments behaves the same either way. + +One assertion therefore covers every harness: + +```json +{ "id": "ran-tests", "type": "transcript_check", + "check": "tool_invocation_matches", "pattern": "Bash.*cargo test" } +``` + +| Harness | Recorded invocation | How it matches | +|---|---|---| +| Claude Code | `Bash {"command":"cargo test"}` | native name | +| Codex | `command_execution {"command":"bash -lc 'cargo test'"}` | `shell` alias `Bash` | +| OpenCode | `bash {"command":"cargo test"}` | `shell` alias `Bash` | +| Cline | `run_commands {"command":"cargo test"}` | `shell` alias `Bash` | + +Evidence keeps the two apart. A native match reads `matched ordinal 4: Bash {"command":"cargo +test"}`. An alias match names the alias and its role, and reports the invocation the harness +actually recorded: + +```text +matched ordinal 4 via shell alias 'Bash': command_execution {"command":"bash -lc 'cargo test'"} +``` + +Two consequences shape how a pattern is written: + +- **Aliases are role-wide.** Within a role any name stands for any other, so `Read` is also + satisfied by a `Glob` call — both are `read` tools. To tell tools inside one role apart, key the + pattern off arguments rather than the name. +- **Undeclared names get no aliases.** A tool the run's harness declares in no role matches by its + native name alone; nothing is invented for it. A custom harness opts in by listing its tool names + under the right role in its descriptor's `[tools]` table — see `eval-magic docs byoh`. + +A miss names the roles that were expanded, so an unexpected zero is readable: + +```text +no candidate matched /Bash|Read/ across 12 invocation(s) (native names plus write/shell role aliases) +``` + +`assistant_message_matches` patterns match message text and are unaffected by any of this. + +## Which evals.json grade reads + +An iteration copies the treatment into its own eval home and stages every condition from that copy, +so what an agent loaded cannot change after the dispatch it explains. Assertions are not the +treatment. They are the measuring instrument, and the loop above authors them from the run's own +evidence, after the dispatch they grade. + +So `grade` splits the file. `assertions` and `skill_should_trigger` come from the live +`/evals/evals.json`, matched per eval id. Everything the run was defined by — `prompt`, +`files`, `turns`, `plan_mode`, `codebase`, `guard`, `runs` — stays as the run captured it. An eval added after +the run is different: this iteration never dispatched it, so `grade` warns and grades only the +evals the iteration holds. + +Every `grade` invocation prints the file its assertions came from: + +``` +Assertions: /path/to/skill/evals/evals.json + refreshed — differs from the run-time copy for 2 eval(s): implement-feature, fix-bug +``` + +Each `grading.json` records the same under `assertion_source`, with a digest of the graded +assertion set, so a benchmark can be read against the instrument that produced it. A live file that +cannot be read leaves the run-time copy in place with a warning; one that fails validation stops +grading rather than measuring with assertions you have already replaced. + +Cached judge responses are keyed by assertion id. Rewording an `llm_judge` rubric under the same id +leaves the previous verdict in place; use `eval-magic dispatch --judges --overwrite` to re-judge it. + +Command-check results use both the authored definition digest and the exact `run.json` digest as +their cache key. `grade` reuses only an exact match. A missing or mismatched digest executes the +check again and replaces the result, so a legacy result, an edited check, or a changed run record +refreshes without `eval-magic grade --overwrite`; that option forces even an exact match to execute +again. + +A command check is eligible only after its task has a runner-owned `run.json`. Partial ingest leaves +an incomplete task environment untouched: no held-out setup, command execution, or result write. If +an incomplete task already carries a cached command-check result, `grade` warns that an older grader +may have contaminated the environment. That task must not be resumed; build a fresh iteration. + +## What the bundle contains + +The bundle combines the evidence that establishes what the agent was asked to do, what it did, and +what it changed: + +- run identity, completion state, timing, token counts, and codebase and skill provenance +- an artifact manifest pointing to `run.json`, `diff-scope.json`, `diff.patch`, and raw harness + outputs +- the original task `prompt` and the agent's `final_message` +- the approved plan, when the eval started in plan mode +- changed-file metrics, a changed-file list, and the captured patch +- the conversation transcript, including markers showing where tools were invoked +- a tool invocation summary with bounded arguments and results + +A one-shot run has an explicit “no conversation record” entry. Missing diff evidence is also +explicit; it is never presented as an empty successful change. + +Held-out `command_check` results are not included. Diff capture happens before command-check setup +files are injected, and keeping the bundle at that boundary prevents a judge from confusing +runner-owned mutations with agent work. Mechanical assertion results remain runner-owned and are +merged during `finalize`. + +## Sample an LLM judge + +An authored `llm_judge` assertion can request several independent verdicts for the same run: + +```json +{ + "id": "clear-review", + "type": "llm_judge", + "rubric": "The review identifies the most important defect and explains its impact.", + "samples": 10 +} +``` + +Use `run --judge-samples N` to set a campaign-wide default. An assertion's `samples` field takes +precedence over that default. The effective count must be at least one. The framework-injected +`__skill_invoked` checks are not substantive grading and remain single-shot per treatment member. + +For a multi-skill treatment, deterministic transcript grading checks every member separately. A +native skill-tool signature compares its identifier with that member's staged slug. An exact-path +access signature instead requires a successful declared read command whose literal argument is +that member's task-specific staged `SKILL.md` path. Failed commands, skill-name text, final-message +phrasing, the live source path, and another treatment member's staged path do not count. A local +result has `confidence: 1.0` and emits no meta judge task. + +The response files use `__skill_invoked__skill-N.json`, and each meta result names its +`skill_name`, so partial and complete access or invocation are distinguishable. A run with no +usable deterministic signature—for example, a descriptor that exposes none—receives one clearly +labeled behavioral-influence fallback task per member. That fallback can estimate whether the +skill influenced the response, but a pass does not prove native invocation or staged-file access. +The compatibility field +`meta_summary.skill_invoked` is true when any treatment member passes its available check. The +`benchmark.json` file retains the suite rate and adds per-skill counts and rates. A scalar treatment +keeps the `__skill_invoked.json` filename and artifact shape. + +Each sample is a separate judge task and response, but every sample for a run receives the exact +same bounded `judge-evidence.md`. The agent is not rerun, and eval-magic does not rebuild or expand +the evidence between samples. This measures agreement among repeated judgments of one execution; +it does not estimate how reliably the agent would succeed across repeated executions. + +For a sampled assertion with `N` verdicts, `grading.json` reports: + +- each verdict in order, including its evidence and confidence +- vote counts and the pass proportion `p = passed / N` +- `pass_power_k = p^N`, the estimated probability that all `N` judgments pass under an + independent-draw assumption + +For example, 6 / 10 passing verdicts produce a vote proportion of `0.6` and pass^k of +`0.6^10`, approximately `0.006047`. This is a stricter judge-consistency endpoint than majority +vote. It is not a statistical significance test. Anthropic's +[eval overview](https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents) explains +the pass^k interpretation in the broader agent-evaluation context. Correlated judge behavior means +`p^N` is a consistency score rather than a calibrated probability, so retain and inspect the +individual verdicts. + +Multi-sample prompts and responses add `__sample-N` to the assertion id in their filenames, and +`judge-tasks.json` records `sample_index` and `sample_count`. `dispatch --judges` skips every +nonempty response independently, so rerunning fills only missing samples. During `finalize`, a +missing response fails that sample and leaves the other samples intact. + +When a run mixes sampled and binary assertions, each authored assertion has equal weight in the +run summary. A binary assertion contributes either 0 or 1; a sampled assertion contributes its +vote proportion to `vote_proportion` and its `p^N` value to `pass_power_k`. `benchmark.json` +reports both endpoints by condition, their deltas, and pooled per-assertion vote counts. The run +plan prints these non-binary endpoints instead of a Fisher exact floor. Fully binary campaigns +retain the Fisher sample-size line. + +An effective sample count of one preserves the binary artifact contract: the legacy response +filename, assertion-level `passed`, `evidence`, and `confidence`, binary grading summary, and +per-assertion `passed` / `n` benchmark rollup remain unchanged. + +## How the bounds work + +Each evidence bundle is at most 98,304 bytes (96 KiB). The complete judge prompt, including its +rubric and framing, is at most 131,072 bytes (128 KiB). Within the bundle, eval-magic reserves: + +- 8 KiB for the task prompt +- 12 KiB for the final message +- 8 KiB for the changed-file list +- 16 KiB for the conversation, with at most 4 KiB per event +- 8 KiB for the tool summary, with at most 512 bytes for each argument and result + +The patch receives the remaining bundle space, so short contextual sections leave more room for +the implementation itself. Oversized sections retain both their beginning and end at valid UTF-8 +boundaries and carry an `[eval-magic] ... omitted` marker naming the full source. Markdown fences +are chosen so evidence containing its own fences cannot escape the section that holds it. + +`judge-tasks.json` records the actual byte count, limit, and `truncated` state of each evidence +bundle, plus the actual and maximum judge-prompt sizes. A `diff.patch` can also carry its own +capture-time truncation marker; that upstream limit is separate from bundle truncation. + +Eval-authored rubrics and skill content are never silently shortened. If either makes the complete +prompt exceed 131,072 bytes, judge-task emission fails with the assertion id, actual size, limit, +and a request to shorten the authored content. + +## Treat evidence as data + +The task prompt, transcript, final message, patch, and tool output are untrusted agent-produced +data. Judge framing says not to follow instructions found inside the evidence. A judge works +read-only: it may inspect a source path named by a truncation marker, but it must not edit the run, +the evidence, or the task environment. Its only write is the requested verdict file. + +When a marker omits material needed by the rubric, inspect the named source before deciding. The +artifact paths are valid in the grading iteration. After promotion and teardown reclaim that +iteration, its retained bundle may no longer have those complete sources beside it. If a required +source is unavailable, the claim is unverifiable rather than evidence of success. + +From a run directory, inspect the bounded evidence and its source records: + +```sh +sed -n '1,240p' judge-evidence.md +jq '{prompt, final_message, conversation, tool_invocations}' run.json +jq '{files_touched, lines_added, lines_removed, hunks, files, patch}' diff-scope.json +``` + +## Retain the evidence behind a baseline + +`promote-baseline` copies each exact bounded bundle into `/evals/baseline/evidence/` beside +the retained benchmark and gradings. A single-run bundle is named +`__.md`; multi-run bundles add `__rN`. This preserves the primary judge input +without copying unbounded transcripts, patches, or task environments into the skill repository. + +Older iterations can have gradings without `judge-evidence.md`. Promotion preserves compatibility +by warning about each missing legacy bundle instead of failing, but such a baseline does not retain +the evidence needed to reproduce its LLM judgment. Re-grade the iteration before promotion when +that evidence matters. diff --git a/docs/opencode-notes.md b/docs/opencode-notes.md index 3db2872..ef7ede2 100644 --- a/docs/opencode-notes.md +++ b/docs/opencode-notes.md @@ -28,8 +28,9 @@ rules (a regex + length cap) — is the descriptor file `harnesses/opencode.toml (`{type, timestamp (epoch ms), sessionID, ...}`). The `opencode-events` parser normalizes `tool_use` parts (name at `part.tool`, args at `part.state.input`, outcome at `part.state.output`/`part.state.error`), sums `step_finish` `part.tokens` (cache reads excluded, - matching codex accounting), takes the final message from the last `text` part, and measures - duration from the envelope timestamps. The declarative extract tier was not sufficient here: + matching codex accounting), takes the final message from the last `text` part, and derives + fallback duration from the envelope timestamps. Runner-driven eval duration uses the runner's + monotonic subprocess measurement. The declarative extract tier was not sufficient here: tool args nest under `state.input` and timestamps are numbers, not RFC 3339 strings. The native `skill` tool (input `{name}`) is a deterministic skill-invocation event, so `surfaces_skill_invocation = true` with `skill_tool = "skill"` / `skill_arg = "name"` and the @@ -37,7 +38,7 @@ rules (a regex + length cap) — is the descriptor file `harnesses/opencode.toml ids (`bash` = the shell tool id; `edit`/`write` take `filePath`; `apply_patch` takes `patchText` — the shared boundary policy reads all three spellings). - **Dispatch recipes + model flag:** the `[dispatch]` templates run `opencode run --dir - --format json --auto` (plus `[model] flag = "-m"`; they landed together because + --format json --agent build --auto` (plus `[model] flag = "-m"`; they landed together because descriptor validation ties the judge template's `$model_arg` to a declared model flag). Verified against opencode v1.18.3 (`opencode run --help` and `packages/opencode/src/cli/cmd/run.ts`): @@ -48,15 +49,12 @@ rules (a regex + length cap) — is the descriptor file `harnesses/opencode.toml - Headless permission asks are auto-**rejected** unless `--auto` is passed; explicit `deny` rules are still enforced under `--auto`. - `--format json` streams raw JSON events (`tool_use` / `text` / `step_finish` / `error`) to - stdout; there is no `--output-last-message`, so the final message comes from the `text` - events via transcript ingest (the dispatch prompt still asks for `outputs/final-message.md`, - which wins when the agent writes it). + stdout; the final message comes from the `text` events via transcript ingest. - `-m` takes models in `provider/model` format; the value passes through verbatim. - Live-verified on v1.18.3 (one `opencode run` per the recipe + `ingest`, #153): the staged skill is discovered and invoked under its slug via the `skill` tool, and the events file - ingests to a full run record. An operator config with an explicit `edit: deny` rule blocks - the `final-message.md` write even under `--auto` (deny rules stay enforced) — record-runs - then falls back to the transcript's last `text` part, so the run still records cleanly. + ingests to a full run record. An operator config with an explicit `edit: deny` rule remains + enforced under `--auto`; transcript-owned completion does not require an agent-authored file. - **Shadow preflight:** the `opencode-skills` capability scans every root OpenCode discovers skills from and warns at build time when a staged logical skill is also live there — see "Isolating from live skills" below. @@ -65,6 +63,22 @@ rules (a regex + length cap) — is the descriptor file `harnesses/opencode.toml supplies and verifies the native id. Verified against `opencode run --help` on 2026-07-24. - **Write guard:** a project plugin staged at `.opencode/plugins/slow-powers-eval-guard.js` — see "Write guard" below. +- **Plan mode:** `[plan_mode]` maps the planning phase to `--agent plan` and act mode to + `--agent build --auto` — see "Plan mode" below. + +## Plan mode + +Verified against opencode **1.18.10** on 2026-09-02, headless, against a scratch repository with one +bug (free model `opencode/nemotron-3.5-lightning-free`): + +- `opencode run --agent plan` without `--auto` is the read-only phase: the built-in plan agent's + `edit` call failed with `The user has specified a rule which prevents you from using this + specific tool call`, and the working tree stayed clean. `--auto` would approve the asks the plan + agent makes, so it belongs to `act_args` only. +- `opencode run --session --agent build --auto` resumed the same session and edited the file. + `--agent build` is explicit so a resumed session does not inherit the plan agent. +- OpenCode writes no plan file, so the descriptor declares no `[plan_mode.plan_file]`; a plan-mode + eval on OpenCode needs a `responder`, whose `done` verdict in the planning phase approves the plan. ## Write guard @@ -104,9 +118,11 @@ Two boundary notes, shared with the other harnesses' guards: - The marker's sole allowed root is the private task env on every host. Host temp locations such as `/tmp` and `$TMPDIR` remain out of bounds; dispatch prompts direct scratch work to `/tmp/` instead without rewriting `TMPDIR`, `TMP`, or `TEMP`. -- Bash coverage is the shared heuristic denylist (installs, git mutations, redirects, config-dir - tampering): a bare `touch /abs/outside/path` matches no pattern and is allowed — after-the-fact - detection of those is `detect-stray-writes`' job, same as claude/codex. +- Bash coverage is the shared target-aware heuristic. Ordinary installs, builds, tests, and + in-place edits run from the task env; recognized explicit project/output destinations must also + remain there. Output redirects, repository-routing escapes, and remote Git mutations remain + blocked. A bare `touch /abs/outside/path` matches no pattern and is allowed — after-the-fact + detection of those is `detect-stray-writes`' job, same as the other harnesses. One hook-shape caveat: `tool.execute.before` fires for *every* tool (OpenCode has no matcher surface), so each tool call spawns one `eval-magic guard-hook`. Classification stays in the @@ -178,7 +194,7 @@ failed or unparseable probe remains `unknown` rather than guessing. Unique runti probe and are `selected`. The shared banner and `aggregate` validity warnings render this same report; historical unversioned artifacts remain readable. -eval-magic detects but cannot unload these sources, and the generated dispatch recipes never +eval-magic detects but cannot unload these sources, and the dispatch commands the runner spawns never set the `OPENCODE_DISABLE_*` kill switches on the operator's behalf — parity with a real user session matters. The operator-facing recipes — both switches with their exact scopes, and move-or-rename as the only remedy for an `.opencode` root — are in the shipped diff --git a/docs/progressive-enhancements.md b/docs/progressive-enhancements.md index eda9dfb..d9a30e2 100644 --- a/docs/progressive-enhancements.md +++ b/docs/progressive-enhancements.md @@ -7,50 +7,53 @@ > at a harness it doesn't know — user-supplied descriptor files, layering, `harness > list`/`show`/`lint` — is the user-facing [BYOH guide](guides/byoh.md). -Harness compatibility is not a parity checklist to audit — it is **a minimal baseline every harness -satisfies, plus optional enhancements** a harness's adapter opts into. Most missing enhancements -have a documented lower-fidelity fallback. Native conversation resume is the deliberate exception: -an eval that declares scripted `turns` is rejected when the harness cannot preserve one session. +Harness compatibility is not a parity checklist to audit — it is **a runner-ready baseline every +harness satisfies, plus optional enhancements** a harness's adapter opts into. Dispatch and +transcript recovery are mandatory because the runner owns execution and run-record assembly. +Native conversation resume remains optional; an eval that declares scripted `turns` or a +`responder` is rejected when the harness cannot preserve one session. ## One dispatch mechanism -One-shot evals and judges dispatch through the harness CLI, one subprocess per task. Scripted evals -use `eval-magic dispatch-task`, one subprocess per round while resuming one harness-native session -and one `eval_root`. The **generated artifacts are the runtime source of truth** for how to -dispatch: `run` writes `RUNBOOK.md` and `dispatch-manifest.md` carrying the exact per-task recipe -for the selected harness — hand-maintained docs never carry command recipes. Eval-agent environment -defaults are harness-independent: `[dispatch.env]` layers merge by key and repeatable -`run --agent-env KEY=VALUE` entries override them. The resolved map is exported by one-shot and -scripted recipes and recorded in `conditions.json` / `dispatch.json`; judges and runner-owned -`command_check` subprocesses remain separate. +`eval-magic dispatch` runs every task — one-shot, scripted, and judge alike — through the harness +CLI, one subprocess per round, `--jobs` tasks at a time. A scripted eval resumes one harness-native +session across its rounds in one `eval_root`. The **generated artifacts are the runtime source of +truth** for how to dispatch: `run` writes `RUNBOOK.md` and `dispatch-manifest.md` naming the exact +command for the selected harness — hand-maintained docs never carry command recipes. Eval-agent +environment defaults are harness-independent: `[dispatch.env]` layers merge by key and repeatable +`run --agent-env KEY=VALUE` entries override them. The resolved map is recorded in +`conditions.json` / `dispatch.json` and applied by the runner when it spawns each task; judges and +runner-owned `command_check` subprocesses remain separate. ## The baseline contract -A harness qualifies at baseline with no harness-specific code beyond naming itself: +A harness qualifies at baseline when its descriptor provides: 1. **A headless exec command** — some way to invoke the harness with a prompt from a chosen cwd and let it run to completion. -2. **A recoverable final message** — the agent writes `outputs/final-message.md` (the dispatch - prompt asks for this), or transcript ingest recovers it where it is wired. +2. **A parseable transcript** — the command captures the harness's event stream under the task's + round output directory, and a named or declarative reader normalizes a non-empty final response. 3. **`--no-stage` when native staging isn't wired** — each `SKILL.md` is inlined into its dispatch prompt instead of staged for native discovery. That baseline already yields a working eval: `llm_judge` assertions grade soft behavior, runner-owned `command_check` assertions can inject held-out files and execute deterministically, -runner-owned final-environment metrics land in `diff-scope.json`, `diff_scope` assertions gate -files/lines deterministically, and the `detect-stray-writes` post-pass (folded into `ingest`) audits -writes that leave the private task environment. Run records without transcript ingest are assembled -from `outputs/final-message.md` or by hand per `schema/run-record.schema.json`. - -In descriptor terms the baseline is one required field: `label`. Everything else in a harness -descriptor is optional for one-shot evals — an absent field or table gets a working generic fallback, and the `run` -preflight *warns* naming that fallback rather than rejecting (a harness without `skills_dir` -forces `--no-stage`; without a declared guard the run continues unguarded behind the -`detect-stray-writes` audit; requested models without a model flag are recorded as provenance -only). Supported enhancements are provided automatically — the write guard auto-arms wherever a -harness declares one and staging is active (`--no-guard` opts out). Only genuinely contradictory -flag combinations stay errors. A selected eval with `turns` also requires `[conversation]`; no -generic fresh-session fallback can preserve the meaning of a canned reply. +runner-owned final-environment metrics land in `diff-scope.json` with the diff itself in +`diff.patch`, `diff_scope` assertions gate files/lines deterministically, and the +`detect-stray-writes` post-pass (folded into `ingest`) audits +writes that leave the private task environment. `dispatch` records monotonic eval-agent subprocess +duration for every runner-ready harness, and `record-runs` assembles every `run.json` and +`timing.json` from the runner-owned dispatch metadata, completion artifact, and per-round +transcripts. + +In descriptor terms the baseline is `label`, `[dispatch].exec_template`, `[transcript]` with one +primary reader, and `[tools]` beside that reader. `run` rejects a harness that cannot dispatch or +recover transcripts. A harness without `skills_dir` forces `--no-stage`; without a declared guard +the run continues unguarded behind the `detect-stray-writes` audit; requested models without a +model flag are recorded as provenance only. Supported enhancements are provided automatically — +the write guard auto-arms wherever a harness declares one and staging is active (`--no-guard` opts +out). A selected eval with `turns` or a `responder` also requires `[conversation]`; no generic +fresh-session fallback can preserve the meaning of a follow-up reply. ## Where this lives in code @@ -81,44 +84,58 @@ generic fresh-session fallback can preserve the meaning of a canned reply. - `run_capabilities()` (descriptor table `[run]`) + `harness_run_preflight()` (`src/cli/run/util.rs`) — the `run` preflight: it resolves the guard tri-state (auto-arm when the harness declares a guard and staging is active; `--guard`/`--no-guard` make it explicit), - and undeclared enhancements warn naming their fallback and adjust the options (guard forced - off, missing `skills_dir` forces `--no-stage`, the no-transcript-parser warning scoped to eval - configs that actually use `transcript_check`, missing dispatch recipes noted); only + rejects descriptors without a dispatch command or transcript reader, and adjusts optional + capabilities (guard forced off, missing `skills_dir` forces `--no-stage`); only contradictory flag combinations (`--bootstrap`/`--stage-name` where the descriptor declares them incompatible with `--no-stage`) and an explicit `--guard` on a user-descriptor-only harness reject. ## Runner-owned environment checks are baseline -Every canonical `(eval, condition, run)` gets a distinct `eval_root`. After fixtures, staging, and -guard installation, `run` recreates a runner-owned Git repository at that root, commits the task -state on branch `work`, runs shadow preflight at the resulting repository boundary, and snapshots -the task environment. Git is therefore a runtime prerequisite; each task starts clean and has no -remotes. During `ingest`, before any held-out setup is injected, the runner compares that baseline -with the final environment and writes raw `files_touched`, `lines_added`, `lines_removed`, and -zero-context Myers `hunks` to `diff-scope.json`. Framework artifacts under the task root's -`.eval-magic-outputs/` and runner-owned `.git/` are excluded; nested repository metadata and all -other new files count. `benchmark.json` preserves these metrics per run even without a `diff_scope` -assertion. An assertion may gate `max_files_touched`, `max_lines_changed` (added plus removed), or -both. +Every canonical `(eval, condition, run)` gets a distinct `eval_root` provisioned from its effective +codebase. After overlays, staging, and guard installation, `run` commits the task state, marks it +with `refs/eval-magic/baseline`, and runs shadow preflight at the resulting repository boundary. +Git is therefore a runtime prerequisite; each task starts clean and has no remotes. Nothing writes +into an environment after the ref is written, so it names exactly what the agent started from. + +During `ingest`, before any held-out setup is injected, Git measures the final environment against +that ref. The runner seeds a scratch index from the baseline, brings it up to the working tree with +one `git add`, and diffs the two trees — so creations, modifications, and deletions all fall out of +one pass, and an untracked creation is not missed. Raw `files_touched`, `lines_added`, +`lines_removed`, and zero-context `hunks` go to `diff-scope.json`, alongside the changed-file list; +the diff itself goes to `diff.patch` beside it, capped and marked when a diff exceeds the cap. +`benchmark.json` preserves the metrics per run even without a `diff_scope` assertion. An assertion +may gate `max_files_touched`, `max_lines_changed` (added plus removed), or both. + +**What counts is what Git counts.** The measurement runs under the same rules the baseline commit +was built under: the codebase's own `.gitignore` holds, so a run that compiles does not report its +build output as thousands of touched files, and the `.git/info/exclude` entry keeps framework +artifacts under `.eval-magic-outputs/` out. Paths the runner force-added despite those rules — the +harness config directories and the declared file overlay — are tracked in the baseline and stay +measured. Git indexes no path with a `.git` component, so a nested repository's internals are +invisible, not just the runner-owned root `.git`. Renames are switched off deliberately: a rename is +two touched files, one created and one deleted, which is what the metric has always meant. A binary +file counts as one touched file with no countable lines. This is deliberately a secondary signal: a smaller diff can be focused, but it can also be -incomplete. Pair a scope gate with a correctness assertion. +incomplete. Pair a scope gate with a correctness assertion. The patch is the evidence that closes +that gap — it is what a judge reads to answer whether the work was any good. `command_check` is intentionally not a harness enhancement. `run` detects the assertion before dispatch so it can validate held-out sources before building. After diff-scope capture, `ingest` -copies the assertion's held-out `setup_files` from the skill's `evals/` directory into that root and -executes the trusted command through the platform shell. Root `.git` paths are reserved for both -visible and held-out fixtures, while nested repositories remain valid. The runner clears inherited -Git routing variables before optional `env` values override the environment; optional `matrix` -values execute every Cartesian-product cell and persist per-cell results. The files are never staged -or mentioned to the agent, and therefore never inflate scope metrics. - -This path needs no transcript parser, tool vocabulary, model flag, or judge recipe, so it behaves -the same for built-ins and descriptor-only harnesses. It also does not use harness tools: an armed -agent write guard can remain installed while the runner executes the command. `finalize` converts -the schema-gated intermediate result into an ordinary grading result, leaving aggregation -harness-agnostic. +grades only tasks with a runner-owned `run.json`, copies the assertion's held-out `setup_files` from +the skill's `evals/` directory into that root, and executes the trusted command through the platform +shell. Root `.git` paths are reserved for both +visible overlays and held-out setup files, while nested repositories remain valid. The runner clears +inherited Git routing variables before optional `env` values override the environment; optional +`matrix` values execute every Cartesian-product cell and persist per-cell results. The files are +never staged or mentioned to the agent, and therefore never inflate scope metrics. + +Beyond the runner-ready dispatch and transcript baseline, this path needs no additional parser +signals, tool vocabulary, or model flag, so it behaves the same for built-ins and descriptor-only +harnesses. It also does not use harness tools: an armed agent write guard can remain installed while +the runner executes the command. `finalize` converts the schema-gated intermediate result into an +ordinary grading result, leaving aggregation harness-agnostic. ## The enhancements @@ -137,9 +154,12 @@ stream needing cross-event state — keyed `tool_use`/`tool_result` joins, shape coercion — is real per-harness code behind a named capability. If a stream needs more than the extract primitives, it's a code capability, not a bigger DSL. -*What it unlocks:* `transcript_check` assertions, token/duration capture, automatic -`run.json`/`timing.json` assembly by `ingest`, and — where the transcript exposes a skill-tool -event — a deterministic `__skill_invoked` meta-check. +*What it unlocks:* `transcript_check` assertions, token capture, historical duration fallback, +automatic `run.json`/`timing.json` assembly by `ingest`, and — where the transcript exposes a skill-tool +event — a deterministic `__skill_invoked` meta-check. Paired with `[tools]`, tool patterns are +portable: grading identifies a native tool name's role from the run's own descriptor and retries +the pattern against every spelling `all_tool_vocabulary()` declares for that role, so one authored +assertion measures the same behavior on every harness (#308). **Sub-capability: permission-denied tool results.** A refused tool call can be reported in the event stream or a paired harness capture while the overall dispatch still exits 0. On its own that @@ -165,14 +185,17 @@ write guard denies through the same permission mechanism, so its blocks land in they are attributed by the `eval guard: ` reason prefix and excluded from the warning so one denial is not reported twice. -*Fallback:* `transcript_check` grades as *unverifiable*, `llm_judge` and runner-owned -`command_check` carry the grading (bias suites toward those for such a harness), tokens/duration go -unrecorded, records are assembled from `outputs/final-message.md` or by hand, and the meta-check -uses the LLM-judge fallback. +There is no no-transcript fallback for a runner-ready harness. A descriptor without a primary +transcript reader is rejected before the workspace is built. A valid reader may still omit optional +token, duration, denial, or skill-invocation evidence; those individual signals remain unavailable +or use their documented grading fallback. *Descriptor fields:* the `[transcript]` table — `events_filename` (gate: an absent table means the ingest pipeline never reads a transcript), one primary summary reader, and -`surfaces_skill_invocation`. The primary reader is either `parser` or the summary outputs under +`surfaces_skill_invocation`. A deterministic native skill event uses `skill_tool` / `skill_arg`; +a successful exact-path shell read uses the mutually exclusive `skill_access` table with its tool, +command argument, exit-code argument, and declared read-command basenames. The primary reader is +either `parser` or the summary outputs under `extract`; validation rejects both, neither, and a surface-only extract. The `extract` sub-table is the declarative tier: equality `where` filters, final and ordered assistant-text picks, a session-id pick, flat tool-item mapping, token sum/subtract reduction, duration rule, and the auxiliary @@ -191,14 +214,34 @@ combination. *Why harness-specific:* each CLI spells same-session continuation differently and exposes its session identifier in a different transcript event. -*What it unlocks:* an eval's ordered `turns` array. `dispatch-task` starts the normal one-shot -command, extracts the native session id, evaluates `agent_asks` (`?`) plus the optional response -regex, and resumes the same session for each delivered follow-up. It writes raw round transcripts -under `outputs/turn-N/` and atomically commits `conversation.json` only after a complete or normal -guardrail-stopped scenario. `ingest` skips an interrupted task with no completion artifact. - -*Fallback:* none. `run` rejects selected multi-turn evals when the harness omits this capability; -silently starting a fresh session would make the canned user response meaningless. +*What it unlocks:* an eval's ordered `turns` array **and** its `responder` policy. `dispatch` starts +the normal one-shot command, extracts the native session id, asks the eval's turn source what +follows each round, and resumes the same session for each delivered follow-up. It writes raw round +transcripts under `outputs/turn-N/` and atomically commits `conversation.json` only after a complete +or normal guardrail-stopped scenario. `ingest` skips an interrupted task with no completion +artifact. + +A scripted turn is gated by `agent_asks` (`?`) plus the optional response regex. A responder instead +*derives* each turn by consulting a small model, once after every round, and records that origin on +the turn itself. **The responder needs no descriptor field and no named capability of its own:** it +reads the round's last assistant message out of `final_text`, which every transcript parser already +normalizes, and it dispatches its own consultations through the same `[dispatch].exec_template` a +judge uses. Every harness that resolves a resume template gets it for free, and none can be +"missing" it. + +That portability is not a happy accident, it is forced. A dispatch runs headless with stdin +detached, so a harness-native question tool has no channel to be answered on; the runner can only +send free text as the next user turn. Text is therefore the only mechanism that fits, and it is the +one every transcript parser already normalizes into `final_text`. + +A consultation binds the exec template's placeholders the way a judge dispatch does — guard +arguments off, its own capture directory, its own prompt — with one addition: `` is the +run's `responder/turn-N/` directory rather than the task env. A consultation must not be able to +write into the codebase under measurement, nor inherit that codebase's `CLAUDE.md` as instructions +to itself. + +*Fallback:* none. `run` rejects selected multi-turn evals — scripted or responder-driven — when the +harness omits this capability; silently starting a fresh session would make the answer meaningless. *Descriptor fields:* `[conversation].resume_exec_template`, with required ``, ``, `{session_arg}`, and `{prompt_arg}` placeholders, plus optional @@ -310,16 +353,19 @@ global `.opencode`, `.claude`, and `.agents` skill dirs — including skills ins harnesses. A logical eval skill present in any such source can contaminate the with/without comparison when dispatches load that source, even when the staged copy uses a unique slug. -*What it unlocks:* a build-time contamination warning (shared banner + schema-v2 +*What it unlocks:* a build-time contamination warning (shared banner + schema-v3 `plugin-shadow.json` in the iteration dir), which `aggregate` folds into `benchmark.json` validity warnings. The runner scans every matrix environment and the shared policy groups scanner -facts by logical skill, records live/staged sources and affected cells, and assigns role-aware -severity. Subject and asymmetric sibling collisions invalidate the comparison; symmetric sibling -collisions warn. Because the scan runs before dispatch it reports *risk*, so the banner states the -consequence conditionally; the verdict is settled afterwards by the session-surface sub-capability -below. When the resolved descriptor declares `isolates_live_sources = true`, the scan, intrinsic -severity, and artifact are retained, but the banner becomes an informational notice and `aggregate` -omits the findings from validity warnings. Historical unversioned artifacts remain readable. +facts by logical skill and source class, records live/staged sources and affected cells, and assigns +role-aware severity. `operator-environment` findings come from inherited global/plugin sources; +`codebase-sourced` findings come from project roots the harness descriptor declares. Subject and +asymmetric sibling collisions invalidate the comparison; symmetric sibling collisions warn. +Because the scan runs before dispatch it reports *risk*, so the banner states the consequence +conditionally; the verdict is settled afterwards by the session-surface sub-capability below. When +the resolved descriptor declares `isolates_live_sources = true`, operator-source scan facts, +intrinsic severity, and artifact are retained, but the banner becomes informational and `aggregate` +omits those findings. Codebase findings use the eval's separate `exclude_skill_sources` policy and +remain warnings when preserved. Schema-v2 and historical unversioned artifacts remain readable. ### Session surface (sub-capability of transcript ingest) @@ -375,37 +421,51 @@ obtained from a best-effort `opencode debug skill` probe only for duplicate runt scan does not enumerate bundled system skills (no stable listing exists), and OpenCode's does not scan config-declared `skills.paths`/`skills.urls` sources. -### Plan-mode context +### Native plan mode + +*Why harness-specific:* the read-only planning mode is the harness's own — a permission mode for +Claude Code, a built-in agent for OpenCode — and so is the way the agent presents its plan. + +*What it unlocks:* evals that declare `plan_mode: true`. The driver dispatches the opening round +with the planning arguments, lets the agent present a plan, approves it with one fixed message, and +resumes the same session with the act arguments; the eval's `turns` or `responder` then proceed as +usual. The approved plan is saved as `outputs/plan.md` and rendered in the judge evidence bundle. +`plan_file` is the deterministic signal that the plan was presented; without one the eval's +responder decides, which is why `run` requires a responder on a harness without a plan file. -*Why harness-specific in principle:* a harness could inject a real native plan mode. +*Fallback:* none. `run` rejects a plan-mode eval for a harness without `[plan_mode]`, before any +environment is built, the way it rejects multi-turn evals for a harness without `[conversation]`. -*What the default does:* wraps the shared `profiles/shared/plan-mode.md` procedure in a -`` block — an approximation that is the same for every harness today, since plan -modes can't be reproduced exactly in a one-shot dispatch anyway. +*Descriptor fields:* the `[plan_mode]` table — `plan_args` and `act_args` fill the `{mode_args}` +slot that both `dispatch.exec_template` and `conversation.resume_exec_template` must carry (the act +arguments also render for judge, responder, and probe dispatches); the optional `[plan_mode.plan_file]` +table names the file the harness writes its plan to (`root`, `~`-expanded, and the write tool's +`content_field`). Writes under that root are allowed by the write guard and the stray-write audit. +Requires `[conversation]`. Validation rejects a table without the slot in both templates, and a +slot with no table to fill it. -*Descriptor fields:* none yet — this is the trait default (`render_plan_mode_context`) with no -descriptor surface; a harness with a real native plan mode would grow one. +*Capability:* `plan-mode` in `harness list`. Claude Code and OpenCode declare it; Codex `exec` has +no plan-mode flag and Cline cannot resume a session, so neither does. -### Dispatch recipes +### Dispatch commands -*Why harness-specific:* the copy-pasteable command template is the harness's CLI. +*Why harness-specific:* the command line the runner spawns is the harness's CLI. -*What it unlocks:* `RUNBOOK.md`, `dispatch-manifest.md`, and the post-`run`/post-`ingest` handoffs -carry exact per-task commands (including parallel and judge variants). +*What it unlocks:* `eval-magic dispatch` itself. Without an `exec_template` there is nothing for the +runner to run, and `dispatch` fails for that harness. -*Fallback:* the generic handoff text; the operator constructs dispatch commands themselves. The -`run` preflight warns naming this limitation when the descriptor declares no `exec_template` -(`has_dispatch_recipes()`). +*Fallback:* none. The `run` preflight rejects a descriptor without `exec_template`, so the gap +surfaces before a workspace is built. -*Descriptor fields:* the `[dispatch]` table — `env`, `exec_template`, `parallel_command_template`, -`judge_command_template`, `next_steps_template`, `manifest_template`, `capture_prefix`, -`guard_args`, `model_note`. Templates carry `{model_arg}`/`{guard_args}` slots the renderer fills -for eval-agent dispatches; judge commands deliberately render empty `guard_args` because they run -outside the guarded task envs. `env` contains non-secret eval-agent defaults; the shared renderer -adds sorted, shell-quoted exports to one-shot, parallel, resume, and probe commands. Unset keys -inherit the host environment and no timezone default is imposed. The shared jq/xargs parallel and -judge scaffolds stay code (`src/adapters/cli_command.rs`) with the per-harness command block spliced -in. Validation rejects a template whose placeholder has no backing field. +*Descriptor fields:* the `[dispatch]` table — `env`, `exec_template`, `next_steps_template`, +`manifest_template`, `guard_args`, `model_note`. Templates carry +`{model_arg}`/`{guard_args}` slots the renderer fills for eval-agent dispatches, plus `{mode_args}` +when a `[plan_mode]` table backs it; a judge dispatch +reuses `exec_template` with `guard_args` deliberately empty, because judges run from the iteration +directory outside every guarded task env. `env` contains non-secret eval-agent defaults, applied +per task when the runner spawns the command; unset keys inherit the host environment and no +timezone default is imposed. Validation rejects a template whose placeholder has no backing +field. ## Current support @@ -431,7 +491,7 @@ data rather than a hand-maintained support table. requires it: the notes file is where the don't-guess guardrail's verification evidence lives. 4. Confirm `eval-magic harness list` and `harness show