From 98dad1fe4c08882bb40613e866f23932550e69b4 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Wed, 11 Nov 2026 14:19:46 +0100 Subject: [PATCH 01/17] Grade a skill's cases without the skill, and report which ones still need it `skillscope behavioral` tells you a skill's cases pass. It cannot tell you they pass *because of* the skill. A case any competent agent satisfies stays green forever whatever happens to the skill, and the report reads as a measurement of the skill when it is a measurement of the model. `skillscope staleness` runs a skill's existing behavioral cases twice, once with the skill installed and once with nothing installed, and puts the two runs side by side. It reads the same `evals.json`, so there is no new file to write and no new assertion to learn. Each case comes back as `earns_it`, `harmful`, `unattributed`, `no_signal`, `fails_either_way`, `guards_only` or `ungraded`. Installing a skill does not make an agent use it, so the graded side also records whether the agent reached for it: a case that moved without the skill firing is `unattributed`, not `earns_it`. `--attempts N` runs each case N times a side, and a side counts as passed if any attempt did, which makes a higher N a harder question rather than a steadier reading of the same one. The baseline is only a measurement if the room was empty, so both halves run against a throwaway `CLAUDE_CONFIG_DIR`, one per half. That works only when auth comes from the environment, so the host engine refuses to start without `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN`, as routing does. An agent reading the skill's source off disk is detected too, and a dirty baseline voids only what it refutes: `no_signal` and `harmful` claim the baseline managed something unaided, so contamination kills them, while `earns_it` and `fails_either_way` rest on the baseline having failed and survive it. A finished run can be read again for free: both halves record which side they were and which invocation wrote them, so `--from-logs` rebuilds the comparison without reaching a model. No verdict fails a run. Two runs of a nondeterministic agent disagree with themselves often enough that a verdict is somewhere to look, not a build failure. An expired deadline or a refused run still exits non-zero. `staleness` is absent from `selection.plan()` and from every workflow, so nothing new runs anywhere and no downstream repo starts running it as a result of this merge. `behavioral`'s JSON gains three additive per-case keys (`activated`, `contaminated`, `attempt`), and behavioral and verify tasks now record which skill and side they were about. Closes #6. --- README.md | 15 +- action.yml | 8 +- docs/authoring-evals.md | 7 + docs/harnesses.md | 5 +- docs/usage.md | 169 ++- pyproject.toml | 2 +- skillscope/behavior.py | 18 + skillscope/cli.py | 643 ++++++++- skillscope/datasets.py | 6 + skillscope/engine/behavioral.py | 340 ++++- skillscope/engine/no_sandbox.py | 12 +- skillscope/engine/verify.py | 66 +- skillscope/staleness.py | 1314 +++++++++++++++++++ tests/test_skillscope.py | 2182 ++++++++++++++++++++++++++++++- 14 files changed, 4720 insertions(+), 67 deletions(-) create mode 100644 skillscope/staleness.py diff --git a/README.md b/README.md index 5db84d9..afdaea5 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,14 @@ SPDX-License-Identifier: MIT ![Skillscope](assets/banner.png) -A test harness for AI agent skills. Install the harness, or point a workflow at your repo. Skillscope enables three main types of tests: +A test harness for AI agent skills. Install the harness, or point a workflow at your repo. Skillscope enables four main types of tests: * **Structural**: Checks whether the skill is well structured (contains the expected files, a proper hierarchy, etc.). * **Routing**: Checks whether the skill triggers when it should and stays quiet when it shouldn't. You may also point to more than one skill here to explore how skills interfere with each other's routing. * **Behavioral**: Runs the agent end to end and uses an LLM judge to confirm the skill behaves correctly. +* **Staleness**: Runs the behavioral cases a second time with the skill withheld, and compares. Tells you whether the cases still need the skill, or whether the model has caught up and the suite is passing on its own. -All three read one dataset per skill, `/evals/evals.json`. How to write one is in [docs/authoring-evals.md](docs/authoring-evals.md). +All four read one dataset per skill, `/evals/evals.json`. How to write one is in [docs/authoring-evals.md](docs/authoring-evals.md). > [!IMPORTANT] > **Early Days**: Flags, workflow inputs, and report formats may evolve quickly as this repo takes shape. @@ -37,6 +38,7 @@ skillscope structural # no agent, no tokens skillscope structural --external # the same, plus checking external URLs skillscope behavioral --skill my-skill # needs an authenticated `claude` CLI skillscope routing --routing-room my-skill,its-neighbour # also needs a key in the environment +skillscope staleness --skill my-skill # the same key: its baseline has to be an empty room ``` `routing` needs `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` set, and refuses @@ -47,6 +49,14 @@ describe a room nobody asked for. `behavioral` installs one skill and grades what the agent produced, so a stray skill is at worst noise there, and a plain `claude login` is fine. +`staleness` is in `routing`'s camp, not `behavioral`'s, and asks the same +credential question on Claude Code. Its baseline half has to be a room with +**no** skills in it, so the runner's own are not noise there — they are the +measurement. Either set one of those variables, or use `--engine +claude-code`, whose guest has no user-level skills at all. It is Claude Code +only: unlike `routing`, it refuses `--harness` outright rather than per +harness, because there is no staleness leg for another one yet. + ## Run it from your CI Simply add an `evals.json` file to your skill and a workflow that points to our reusable workflow. This will triger all graders to run directly from your repo. @@ -72,6 +82,7 @@ You may also choose to customize or even disable some tests if you prefer it. Se | `skillscope structural` | Skill folders, datasets, and markdown references. | | `skillscope routing` | Which skill fires, with several installed together. | | `skillscope behavioral` | What a skill does once it has fired. | +| `skillscope staleness` | Whether those cases still need the skill. | | `skillscope select` | The CI plan for a change, as JSON. | | `skillscope list-skills` | The skills that have a dataset, as JSON. | | `skillscope template` | The dataset template a new skill starts from. | diff --git a/action.yml b/action.yml index 2a73c12..721ef6c 100644 --- a/action.yml +++ b/action.yml @@ -16,7 +16,7 @@ description: >- inputs: command: - description: 'The skillscope subcommand: structural, routing, behavioral, select, list-skills, template.' + description: 'The skillscope subcommand: structural, routing, behavioral, staleness, select, list-skills, template.' required: true args: description: "Further arguments for the subcommand, shell-quoted." @@ -47,8 +47,10 @@ inputs: default: "true" install-claude: description: >- - Install Node and the claude CLI. Needed by `routing` and `behavioral`, - pointless for `structural` and `select`. + Install Node and the claude CLI. Needed by `routing`, `behavioral` and + `staleness` -- staleness drives the behavioral leg for both of its + halves, so it needs the CLI exactly as much. Pointless for `structural` + and `select`. required: false default: "false" node-version: diff --git a/docs/authoring-evals.md b/docs/authoring-evals.md index 25a6b19..ee786bb 100644 --- a/docs/authoring-evals.md +++ b/docs/authoring-evals.md @@ -224,8 +224,15 @@ skillscope structural # your folder, prose, and da skillscope behavioral --skill # your skill, end to end skillscope routing --routing-room , skillscope routing --only --keep-logs logs # one case, keeping the transcript +skillscope staleness --skill # do your cases still need the skill? ``` +`staleness` is the one aimed at this file's subject. It runs your behavioral +cases twice, once with the skill installed and once without, and reports which +of them passed either way. Those are cases that are not evidence about the +skill — either it has stopped mattering, or the case never tested it. It costs +two runs, so it belongs on a schedule rather than in the loop. + `structural` is instant and needs no agent. Everything else needs the `claude` CLI authenticated, plus whatever your own cases need. Flags, what each check asserts, the routing bar, and how to read a report are in diff --git a/docs/harnesses.md b/docs/harnesses.md index 4c3be4b..a8ad233 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -145,4 +145,7 @@ Claude Code, and a harness other than it cannot yet use them: each agent is defined once. Until it does, a harness other than Claude Code has no sandboxed leg; * **`.claude/skills`** as the path routing treats as a skills inspection, rather - than a harness's own `skills_dir`. + than a harness's own `skills_dir`; +* **`staleness`**: its baseline is only a measurement if the room is empty, and + the check that establishes that reads Claude Code's credentials and no other + harness's, so it refuses `--harness` outright rather than per harness. diff --git a/docs/usage.md b/docs/usage.md index ee4c527..1de7a86 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -18,6 +18,7 @@ CLI flag, and every workflow input. For writing a skill's dataset, see * [References](#references) * [Routing, and the bar it is held to](#routing-and-the-bar-it-is-held-to) * [Hardware a skill needs](#hardware-a-skill-needs) +* [Is the skill still earning its place?](#is-the-skill-still-earning-its-place) * [In CI: one job](#in-ci-one-job) * [In CI: the full pipeline](#in-ci-the-full-pipeline) * [Versions](#versions) @@ -249,6 +250,164 @@ Legs with a scoped environment run as a separate job, because a job's credentials are fixed before its matrix expands. A repo that declares no scoped environment gets one matrix, labels and all. +## Is the skill still earning its place? + +A behavioral run says a skill's cases pass. It cannot say they pass *because +of* the skill, and those are different claims: an expectation any competent +agent meets is a measurement of the model with the skill's name on it. + +`skillscope staleness` runs the same cases twice, once with the skill +installed and once with nothing installed, and reports which ones moved. + +```bash +skillscope staleness --skill my-skill +``` + +On the default engine this needs `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` +in the environment and refuses to start without one, the same way `routing` +does and for a stronger version of the same reason: the key is what lets the +run redirect the CLI away from the runner's own config dir, and the baseline +half has to be a room with no skills in it. Without it, every skill installed +on the machine joins the baseline, and a case one of them solves there is +reported as a case the skill under test was not needed for. A plain `claude +login` is enough for `behavioral` and is not enough here. `--engine +claude-code` needs no key for this, since its guest has no user-level skills +at all. + +| Verdict | Means | +| --- | --- | +| `earns_it` | passed with the skill, failed without it, and the skill fired — the case discriminates | +| `harmful` | failed with the skill, passed without it — the skill steered the agent wrong | +| `unattributed` | passed only with the skill installed, but the skill never fired on the attempt that passed | +| `no_signal` | passed either way — says nothing about the skill | +| `fails_either_way` | failed both times — a broken case, or a hard one | +| `guards_only` | asserts nothing the skill could be credited for — see the guard rails below | +| `ungraded` | nothing could be concluded: every attempt errored on one side or both, a case was missing from a side, its baseline was not empty, or whether the baseline was empty could not be established | + +A verdict says what changed, not why. So the graded side also records whether +the agent *reached for* the skill, using the same detector the routing leg +uses: installing a skill does not make an agent use it, and "installed and the +outcome differed" is a weaker claim than "the skill fired and the outcome +differed". A case that moved without the skill firing is `unattributed` rather +than `earns_it`: it stays out of the headline number and is reported as the +lead it is. Guard rails -- `unexpected_behavior` -- are kept +out of the verdict entirely, since a good one holds whether or not the skill is +installed; they are counted on their own, where a guard broken *with* the skill +installed is the serious finding. + +The verdict is binary and the evidence behind it is not, so the report prints +both. Every case carries how many attempts passed on each side, and how much +of the case each side satisfied on average: `fails_either_way` covers a +baseline that met four claims of five and one that met none, and only one of +those says the case is worth keeping. + +`no_signal` is the verdict worth acting on, and it is two findings wearing one +name: either the skill has stopped mattering, or the case was never evidence +about it in the first place. The second is the more common, which makes this a +check on the dataset as much as on the skill. + +Above one attempt there is a third, and the report separates it out. A case +where a side passed some of its attempts and failed others is `no_signal` +because the baseline managed it at least once, and that is not the same +finding as a case both sides managed every single time. One says the agent +does this unaided; the other says the two sides did not settle. The advice +column says which, and the counts beside it are the evidence: act on the +first, re-run the second. + +### Spending less on a run + +Both halves of a comparison are agent passes, and the baseline is the slower +one: an agent holding the skill goes straight to the work, and an agent +without it explores. Three flags decide what a run costs. + +| Flag | Default | Effect | +| --- | --- | --- | +| `--attempts N` | 1 | Run every case N times on each side. Costs N times as much. A side counts as having passed when **any** of its attempts did, so a higher N is a harder question rather than a steadier reading of the same one: `earns_it` at three attempts needs the baseline to have failed all three times, and a baseline that passes once in three takes the case to `no_signal`. Verdicts compare with verdicts from runs at the same N, not across different ones. What you gain is the counts printed beside each verdict: `3/3` against `0/3` is a result, `3/3` against `1/3` is a baseline that managed it once, and `2/3` against `1/3` is two sides that both disagreed with themselves. The report calls that last one out rather than letting it read as a dead case. | +| `--max-cases N` | every case | Grade only the first N cases of each selected skill, after `--only` and `--extended` have chosen them. Both sides run the same N, and the report says the run did not cover the whole suite. | +| `--concurrency N` | inspect's default | How many cases may be in flight at once. Where the wall clock goes on the baseline half. Raise it as far as the engine's sandboxes and the model's rate limits allow. | + +None of the three reaches a `--from-logs` rebuild, which runs no agent, and +passing one with it is refused rather than ignored. `--no-extended` is refused +with it for the same reason: which evals were graded is a property of the run +that wrote the logs. `--skill` and `--only` do work on a rebuild, where they +narrow what is reported rather than what is run. + +Both halves write which side they were, whose skill they were about, where that +skill was on disk, and which invocation wrote them into their inspect log +metadata (`skillscope_baseline`, `skillscope_skill`, `skillscope_skill_dir`, +`skillscope_staleness_run`). So a finished run can be looked at again without +paying for it twice: + +```bash +skillscope staleness --from-logs .skillscope/logs +``` + +That reaches no model and runs no preflight, having nothing to spend. + +A rebuild reads **one run per skill, the most recent**. Nothing prunes +`.skillscope/logs`, so after your second staleness run the directory holds two +of everything, and reading both would report every case once per run and count +it that many times in the totals: a one-case skill would come back as a +two-case suite with the same row printed twice. The report names the run it +read and lists the ids it passed over. `--run-id` reads one of those instead: + +```bash +skillscope staleness --from-logs .skillscope/logs --run-id 7f3a9c21 +``` + +Every key is load-bearing. `skillscope_skill_dir` is what lets a rebuild judge +whether the baseline was really empty: the check recognises a tool call that +reached the skill by comparing against the skill's absolute path, so a reader +that worked that path out for itself would be working out its own rather than +the run's, and would report a baseline the run had voided as a clean one. A log +written without the key is read as an unknown room rather than an empty one. + +The run id is the key that makes this safe to point at `.skillscope/logs`. That directory is shared with `behavioral` +and `routing`, is gitignored wholesale, and nothing prunes it — and every +ordinary `behavioral` run writes the same `skillscope_baseline: false` a graded +half does, with no run id. A plain `behavioral` log is therefore shaped exactly +like a graded half, and always will be, since only `staleness` mints an id. +Pairing on the skill alone would splice an unrelated run from last month in as +this comparison's "with skill" side, under a table that reads exactly as +confidently as a real one. + +So logs carrying no run id are passed over, and a directory of plain +`behavioral` logs rebuilds as nothing rather than as a comparison. Logs written +before the id existed are passed over for the same reason; `--unlabelled-logs` +reads them: + +```bash +skillscope staleness --from-logs .skillscope/logs --unlabelled-logs +``` + +That asserts the directory holds nothing else for the skills in it. Even then, +a group of unlabelled logs needs a baseline half, which nothing but `staleness` +ever writes, so ordinary `behavioral` logs stay out whatever flags you pass. +Where one side holds several unlabelled logs there is a choice to make and +nothing to make it with, so every case in that group is reported `ungraded`. + +The model and effort are not reported for a rebuild: nothing reads them off +the logs, and the logs need not agree on them. + +Budget more wall clock than a `behavioral` run of the same cases, and more +than double it. `--timeout` defaults to twice what the single-pass commands +use for that reason. Both passes share one `--timeout`, and the baseline is +reliably the slower half: the agent holding the skill goes straight to the +work, while the one without it explores. On a two-case skill the graded pass +took 2m23s and the baseline had not finished its second case half an hour +later. A bound that expires partway reports the unreached cases as `ungraded`, +after the graded half has already been paid for. + +No verdict it reaches fails a run, for the reason the `claude-code` leg does +not gate either: the agent is nondeterministic on both sides, so a single pair +is a place to look rather than a verdict. An expired `--timeout` or a refused +run still exits non-zero, because those are statements about the run rather +than about a skill. [`tools/benchmark_engines.py`](../tools/benchmark_engines.py) +is the shape of the honest version of this comparison — it measures agreement +against how much one engine already disagrees with itself — and a staleness +number deserves the same scepticism. It also costs twice what `behavioral` +costs, which is why no workflow runs it for you. + ## Which engine grades a run Two choices decide what runs the cases, and they are separate: `--engine` is @@ -304,8 +463,14 @@ routing leg at all unless one of the harness's credential variables is set (see [harnesses.md](harnesses.md): `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` for Claude Code, `COPILOT_GITHUB_TOKEN`, `GH_TOKEN` or `GITHUB_TOKEN` for Copilot CLI), which is what lets it redirect the CLI's -config dir away from the runner's own. Refusing beats being -quietly wrong about every case. +config dir away from the runner's own. + +`staleness` is stricter, and not on the same terms. It has no leg for any +harness but Claude Code yet, so it refuses `--harness` outright whatever +credentials are present, rather than scaling the refusal per harness the way +routing does. On Claude Code it then asks routing's question as well, because +there the empty room is not one input to the measurement, it is the +measurement. Refusing beats being quietly wrong about every case. Both legs stop at the decision, by different routes. `claude-code`'s calls cross inspect's bridge, so the one that reveals the decision is declined diff --git a/pyproject.toml b/pyproject.toml index 70a26a8..0544635 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ authors = [{ name = "Advanced Micro Devices, Inc." }] # # It was an extra while `legacy` existed: that engine was standard library only, # so a repo that had not migrated installed nothing new. `legacy` is gone, and -# keeping the extra would ship a tool whose two graded commands both fail on a +# keeping the extra would ship a tool whose graded commands all fail on a # plain `pip install` -- which is what CI found, by running the suite without # extras and getting `ModuleNotFoundError: inspect_ai` from the default engine. # diff --git a/skillscope/behavior.py b/skillscope/behavior.py index b382b1d..1b0caf3 100644 --- a/skillscope/behavior.py +++ b/skillscope/behavior.py @@ -29,11 +29,29 @@ class BehaviorOutcome: elapsed_s: float checks: list[dict] = field(default_factory=list) error: str | None = None + # The skill the agent actually reached for, if any. A behavioral run + # installs a skill but cannot make the agent use it, and until this was + # recorded a case that passed was credited to a skill that may never have + # fired. `skillscope staleness` is what the distinction is for: "installed + # and the outcome differed" is a weaker claim than "the skill fired and the + # outcome differed", and only the second is evidence about the skill. + activated: str | None = None + # Why this run was not the room it claimed to be: another skill fired, or + # the agent read the skill's own source off disk. Harmless noise in a + # behavioral run, which grades what the agent produced. Fatal to a + # staleness baseline, whose whole claim is that nothing was available. + contaminated: str | None = None # Set when the failure was the model provider's rather than the skill's. # A behavioral case that failed on a gateway 504 is a skill nobody # measured, and reporting it beside one that genuinely failed its # expectations invites exactly the wrong conclusion. degraded: bool = False + # Which run of this case this is, where `skillscope staleness --attempts` + # asked for more than one. Always 1 elsewhere. It exists so two answers to + # the same case can be told apart: `staleness` groups a side's attempts by + # case id, and the guard that refuses a case two logs both claim to answer + # has to know a repeat from a collision. + attempt: int = 1 # -------------------------------------------------------------------------- diff --git a/skillscope/cli.py b/skillscope/cli.py index bc9c406..90817c3 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -5,7 +5,7 @@ """The one entry point for skill evals. Every skill owns a single dataset at ``/evals/evals.json``. This runner -reads those datasets and grades them with three commands: +reads those datasets and grades them with four commands: * ``structural`` -- every skill folder, every dataset, and every reference the skill's markdown makes. No agent, no tokens, instant. @@ -18,13 +18,17 @@ grades ``expected_behavior`` / ``unexpected_behavior`` / ``logs_contain`` / ``files_exist``. Only runs evaluations that assert something beyond the routing decision. + * ``staleness`` -- runs ``behavioral``'s cases twice, once with the skill + installed and once without, and compares. Answers a question the other + three cannot: whether the cases still need the skill, or whether the model + has caught up and the suite is passing on its own. A skill may also ship ``/evals/extended_evals.json``, in the same format -and under no coverage requirement of its own. Both graded commands include it +and under no coverage requirement of its own. The graded commands include it by default; ``--no-extended`` grades the required dataset alone. -The prompt is written once and both graded commands read it, which is the whole -point: the alternative is a central routing prompt set plus a separate +The prompt is written once and every graded command reads it, which is the +whole point: the alternative is a central routing prompt set plus a separate per-skill test file that re-asserts routing with a substring match on the transcript. @@ -52,6 +56,13 @@ # one case, keeping the raw transcript skillscope routing --only qwen-on-mi300x --keep-logs eval-logs + # whether the skill is earning its place: every case run with it and + # without it, twice the agent passes of a behavioral run + skillscope staleness --skill serving-llms-on-epyc + + # the same comparison again, read back off the logs it already wrote + skillscope staleness --from-logs .skillscope/logs + # what CI should run for a change skillscope select --since BASE HEAD @@ -68,7 +79,9 @@ import json import os import sys +import tempfile import time +import uuid from pathlib import Path from . import ( @@ -79,6 +92,7 @@ engine, references, routing, + staleness, structure, usage, ) @@ -442,7 +456,7 @@ def _fail_if_expired() -> int | None: def _prepare_graded_run( args: argparse.Namespace, scope: list[str] | None = None ) -> list[str]: - """Structural checks, model pin, and API reachability. Shared by both graders. + """Structural checks, model pin, and API reachability. Shared by the graders. The structural gate reads `scope`, defaulting to the skills ``--skill`` selected -- the ones about to be graded, and nobody else. A routing run @@ -484,8 +498,8 @@ def _prepare_graded_run( ) # Also before the probe, and for the same reason: a malformed price # table should fail the command, not one sample into a graded run. - # Registered here rather than in the engine because both graded - # commands come through this function and inspect's registry is + # Registered here rather than in the engine because every graded + # command comes through this function and inspect's registry is # process-global -- setting it twice would be the same work. from .engine import pricing as engine_pricing @@ -537,9 +551,11 @@ def _require_hook_support(engine: str, skills: list[str], command: str) -> None: run perfectly well. Routing is exempt on every engine: it installs the skills, asks which one - fires, and executes nothing. + fires, and executes nothing. Staleness is not: it drives the same + `build_task` on the same cases, twice, so a `check` it skipped is skipped + on both halves and the comparison is between two runs that both lied. """ - if command != "behavioral": + if command not in ("behavioral", "staleness"): return from .engine import hooks as engine_hooks @@ -784,61 +800,143 @@ def cmd_routing(args: argparse.Namespace) -> int: ) -def cmd_behavioral(args: argparse.Namespace) -> int: - skills = _prepare_graded_run(args) - subject = harnesses.get(args.harness) - started = time.time() +def _gradable_cases(args: argparse.Namespace, skills: list[str]) -> list: + """The cases a behavioral-style run grades: selected, filtered, assertive. + Shared so `behavioral` and `staleness` cannot drift apart about what + `--only` or `--extended` select. A staleness run comparing a different set + of cases than the behavioral run it is read beside would be the one bug + neither report could show you. + """ cases = [] for skill in skills: cases.extend(datasets.load_dataset(skill, extended=args.extended)) if args.only: + # Applied across every selected skill at once, because it refuses a + # selection that matched nothing. Per skill, `--only` naming a case of + # one skill would fail the run on the next skill's empty match. cases = datasets.filter_cases(cases, args.only) - gradable = [c for c in cases if c.has_behavior] - - if not gradable: - print( - "[behavioral] no evaluation in the selected skill(s) asserts " - "anything beyond routing, so there is nothing to grade. Add " - "`expected_behavior` / `unexpected_behavior` / `logs_contain` / " - "`files_exist` to a triggering evaluation." - ) - return 0 + cases = [c for c in cases if c.has_behavior] + # Capped here rather than through inspect's own `limit`, which applies per + # task, and a run builds one task per skill: the same number would mean + # "the first N cases" on a one-skill run and "N per skill" on a four-skill + # one. Capped after the filters, so `--only` and `--max-cases` compose the + # way a reader would expect. + # + # And capped once, in the shared selector, because the two halves of a + # staleness run have to grade the same cases. A cap applied anywhere + # downstream could hand the baseline a different set, and a comparison + # between two runs that answered different questions is worse than no + # comparison: nothing in the report would say so. + # + # Per skill rather than over the whole run, so a four-skill run does not + # spend its whole budget on the first skill's dataset and report the other + # three as having nothing to grade. + cap = getattr(args, "max_cases", 0) + if cap: + kept: list = [] + for skill in skills: + kept.extend([c for c in cases if c.skill == skill][:cap]) + return kept + return cases + + +def _run_behavioral( + args: argparse.Namespace, + skills: list[str], + gradable: list, + *, + install: bool = True, + config_dir: Path | None = None, + run_key: str | None = None, +) -> list: + """Run the behavioral cases on the selected engine. + + `install` False runs them with no skill staged, which is the baseline half + of `cmd_staleness`. Both halves go through here rather than through two + similar call sites, so the only difference between the run being measured + and the run it is measured against is the flag. + + `run_key` is stamped on the logs both halves write, so `--from-logs` can + find this run's two halves and only this run's. Set by `cmd_staleness` + alone -- see `engine/behavioral.py:STALENESS_RUN_KEY`. + + `attempts` and `concurrency` are read off `args` rather than passed, so + the two halves cannot be given different ones. A baseline attempted fewer + times than the graded half is not a baseline, and the counts the report + prints would be comparing different denominators without saying so. + """ from .engine import behavioral as inspect_behavioral from .engine import models as engine_models + attempts = getattr(args, "attempts", 1) + concurrency = getattr(args, "concurrency", 0) + if args.engine == "claude-code": from .engine import verify as runner - outcomes = runner.run( - skills, gradable, engine_models.resolve(args.model), args.effort + return runner.run( + skills, + gradable, + engine_models.resolve(args.model), + args.effort, + install=install, + run_key=run_key, + attempts=attempts, + concurrency=concurrency, ) - elif args.engine == "claude-code-no-sandbox": + if args.engine == "claude-code-no-sandbox": # The real CLI, driven on the host, inside inspect's framework. # `--model` stays the CLI's own alias here: this one does not go # through an inspect model provider. from .engine import no_sandbox + # The harness is read here rather than passed in, for the reason + # `attempts` and `concurrency` are: both halves of a staleness run go + # through this function, so reading it once is what stops the two + # halves being given different agents. + subject = harnesses.get(getattr(args, "harness", None)) no_sandbox.require_local(subject) - outcomes = inspect_behavioral.run( + return inspect_behavioral.run( skills, gradable, engine_models.resolve(args.model), args.effort, solver_factory=lambda d: no_sandbox.harness_solver( - subject, args.model, args.effort, d + subject, args.model, args.effort, d, config_dir, install=install ), + baseline=not install, + run_key=run_key, + attempts=attempts, + concurrency=concurrency, ) - else: - # Not a fallthrough. An engine in ENGINES with no branch here is a - # bug this dispatch has had once already, and a silent default is - # what let it survive: runs asked for one agent, got another, and - # the report named the one they had asked for. - raise SystemExit( - f"error: --engine {args.engine} has no behavioral leg. " - f"This is a bug in skillscope, not in how it was called." + # Not a fallthrough. An engine in ENGINES with no branch here is a + # bug this dispatch has had once already, and a silent default is + # what let it survive: runs asked for one agent, got another, and + # the report named the one they had asked for. + raise SystemExit( + f"error: --engine {args.engine} has no behavioral leg. " + f"This is a bug in skillscope, not in how it was called." + ) + + +def cmd_behavioral(args: argparse.Namespace) -> int: + skills = _prepare_graded_run(args) + started = time.time() + + gradable = _gradable_cases(args, skills) + + if not gradable: + print( + "[behavioral] no evaluation in the selected skill(s) asserts " + "anything beyond routing, so there is nothing to grade. Add " + "`expected_behavior` / `unexpected_behavior` / `logs_contain` / " + "`files_exist` to a triggering evaluation." ) + return 0 + + outcomes = _run_behavioral(args, skills, gradable) summary = behavior.summarize( outcomes, @@ -863,6 +961,330 @@ def cmd_behavioral(args: argparse.Namespace) -> int: return 0 +def cmd_staleness(args: argparse.Namespace) -> int: + """Run every behavioral case twice, with the skill and without it. + + Both passes run under the one `--timeout`, so a deadline that expires + partway leaves the rest of the second pass unmeasured -- those cases are + reported `ungraded` rather than as a skill that made no difference. + + Budget for a baseline that runs *longer than the graded pass*, not the + same. An agent holding the skill goes straight to the work; an agent + without it explores -- building throwaway virtualenvs, dry-running + installs, reading around the problem. Measured on a two-case skill: the + graded pass finished both cases in 2m23s and the baseline was still going + on the second after 30 minutes. Twice a `behavioral` bound is the floor, + not the figure -- which is why this parser defaults `--timeout` to twice + what the single-pass commands use. On the shared 900s default those + numbers left the slower half ~637s of a measured 1800s+ need, so + truncation was the expected outcome on this command's own example rather + than a risk it ran. + + Always exits 0 on a completed run. The comparison is nondeterministic on + both sides, so a verdict here is somewhere to look rather than a build + failure -- see `skillscope/staleness.py`. A deadline that expired still + fails, because that is a statement about the run rather than about a skill. + """ + started = time.time() + + asked = getattr(args, "harness", None) or os.environ.get("SKILLSCOPE_HARNESS") + if not args.from_logs and asked and asked != harnesses.DEFAULT: + # The one place this command cannot be harness-neutral yet. Its + # baseline half is only a measurement if the room is empty, and the + # check that establishes that -- `routing.can_isolate_config` below -- + # reads Claude Code's credential variables and no other harness's. Run + # against a different agent with an Anthropic key happening to sit in + # the environment, it would report an isolated room it had not + # isolated, which is the one error this whole command exists to avoid + # making. Refused rather than run on a baseline nobody can vouch for. + raise SystemExit( + f"error: --harness {asked} has no staleness leg yet. The baseline " + "half has to be a room with no skills in it, and the check that " + f"establishes that reads {harnesses.DEFAULT}'s credentials rather " + "than this harness's -- so a run here would claim an empty room " + f"it never checked. Use --harness {harnesses.DEFAULT}, or " + "`routing` and `behavioral`, which are harness-neutral." + ) + + if args.run_id and not args.from_logs: + # Same habit as `--unlabelled-logs` below. A live run mints its own + # run id and stamps it on both halves, so there is no run here for a + # reader to choose between. + raise SystemExit( + "error: --run-id only means something with --from-logs. A run " + "started now mints its own id, and there is nothing to pick " + "between." + ) + + if args.unlabelled_logs and not args.from_logs: + # The repo's habit: refuse an argument that cannot do anything rather + # than accept and ignore it. A live run writes its own run id on both + # halves, so there is nothing unlabelled for this to reach. + raise SystemExit( + "error: --unlabelled-logs only means something with --from-logs. " + "A run started now stamps a run id on both halves, so it has no " + "unlabelled logs to read." + ) + + if args.attempts < 1: + raise SystemExit( + f"error: --attempts {args.attempts} would run nothing. The " + "command compares a side that ran against a side that ran; one " + "attempt each is the minimum that produces a comparison." + ) + if args.max_cases < 0 or args.concurrency < 0: + raise SystemExit( + "error: --max-cases and --concurrency count things, so they " + "cannot be negative. 0 means no cap on either." + ) + + # Same habit again. None of these reaches a rebuild: `--from-logs` starts + # no agent, and the attempts, the cap, the concurrency and which evals + # were graded are all properties of the run that wrote the logs rather + # than of the reader. Accepting them silently would let somebody believe + # they had re-measured something at three attempts when they had reread + # one attempt. + # + # `--skill` and `--only` are not on this list, and used to be ignored like + # it. They name a subset of what is already there, which a reader of a + # shared log directory has every reason to want, so the rebuild filters on + # them below instead. + spent = [ + name + for name, value in ( + ("--attempts", args.attempts > 1), + ("--max-cases", args.max_cases), + ("--concurrency", args.concurrency), + ("--no-extended", not args.extended), + # The flag, not `asked`: an ambient `SKILLSCOPE_HARNESS` is not an + # argument to this invocation, and refusing a free rebuild over a + # variable the shell happens to export would be a refusal nobody + # asked for. + ("--harness", args.harness), + ) + if value + ] + if args.from_logs and spent: + raise SystemExit( + f"error: {', '.join(spent)} cannot change a --from-logs rebuild. " + "It runs no agent: the attempts, the cases, the concurrency, the " + "agent and which evals were graded are whatever the logs were " + "written with. " + "The report reads them back off the logs. Use --skill or --only " + "to narrow what is reported." + ) + + if args.from_logs: + # Nothing is run, so nothing is prepared: no structural gate, no model + # pin, no reachability probe. Those guard a spend that is not about to + # happen, and demanding them would make a free command fail for + # reasons that cannot affect its answer. + rebuild = staleness.from_logs( + args.from_logs, + unlabelled=args.unlabelled_logs, + run_id=args.run_id or None, + ) + comparisons = rebuild.comparisons + known = sorted({run for ids in rebuild.runs.values() for run in ids}) + if args.run_id and not comparisons and known: + # Told apart from "no staleness logs here at all", because they + # are different mistakes: this one is a typo in an id, and the + # answer to it is the list of ids that would have worked. + raise SystemExit( + f"error: no staleness run `{args.run_id}` in " + f"{args.from_logs}. The runs there are: " + + ", ".join(f"`{run}`" for run in known) + + "." + ) + if args.run_id: + # `--run-id` is global and a run id is not. One invocation mints + # one id and stamps it on whichever skills it was asked for, so + # naming an id silently drops every skill that invocation did not + # cover -- and the guard above cannot catch it, because it only + # fires when *nothing* matched. Asking for one run and getting one + # run is legitimate; losing a skill without being told is not. + covered = {c.skill for c in comparisons} + missing = sorted(set(rebuild.runs) - covered) + if missing: + print( + f"[staleness] run {args.run_id} covers " + f"{len(covered)} of the {len(rebuild.runs)} skill(s) in " + f"{args.from_logs}. Nothing is reported for " + + ", ".join(missing) + + ": that run did not cover them. Each has runs of its " + "own; drop --run-id to read the most recent of each.", + flush=True, + ) + + # `--skill` and `--only` narrow what is reported rather than what is + # read. A log directory is shared and accumulates, so a reader asking + # about one skill in it is the ordinary case; ignoring the flags, as + # this used to, answered a question nobody asked. + wanted = {s.strip() for s in args.skill.split(",") if s.strip()} + cases = {c.strip() for c in args.only.split(",") if c.strip()} + if wanted or cases: + comparisons = [ + c + for c in comparisons + if (not wanted or c.skill in wanted) + and (not cases or c.id in cases) + ] + if not comparisons: + raise SystemExit( + "error: nothing in " + f"{args.from_logs} matches " + + " and ".join( + part + for part in ( + f"--skill {args.skill}" if wanted else "", + f"--only {args.only}" if cases else "", + ) + if part + ) + + "." + ) + if not comparisons: + from .engine import behavioral as engine_behavioral + + raise SystemExit( + f"error: no skillscope staleness logs in {args.from_logs}. " + f"Both halves of a comparison carry `{engine_behavioral.SKILL_KEY}`, " + f"`{engine_behavioral.BASELINE_KEY}` and " + f"`{engine_behavioral.STALENESS_RUN_KEY}` in their eval " + "metadata; logs written by another tool, or by a plain " + "`behavioral` run, cannot be paired." + + ( + "" + if args.unlabelled_logs + else "\n Logs written before the run id existed carry " + "none, and are passed over. `--unlabelled-logs` reads " + "them, where a baseline half is there to pair with." + ) + ) + # No `model`, `engine` or `effort`. Nothing here read them off the + # logs, and `args` holds this invocation's defaults -- which describe + # the command that did the rebuilding, not the runs being rebuilt. The + # report prints what it is given, so passing them through made it + # assert a model the measurement was never taken on. + summary = staleness.summarize( + comparisons, + { + "skills": sorted({c.skill for c in comparisons}), + "from_logs": args.from_logs, + # The runs behind what is being reported, so the report can + # name the one it read and say what it passed over. Narrowed + # to the skills that survived the filters above: a skill the + # reader excluded has runs that were not read, and counting + # those as passed over would answer a question nobody asked + # with a number nobody wanted. See `staleness.Rebuild`. + "runs": { + skill: ids + for skill, ids in rebuild.runs.items() + if skill in {c.skill for c in comparisons} + }, + "run_id": args.run_id or None, + "wall_time_s": round(time.time() - started, 1), + "github_run_id": os.environ.get("GITHUB_RUN_ID"), + }, + ) + _write_report(summary, staleness.render_markdown(summary), args, "staleness") + # The docstring says a deadline that expired still fails, and said it + # of this branch too while this branch alone returned without asking. + # Reading a large directory is not free, and a command that overran + # the bound it was given should say so whatever it spent the time on. + return _fail_if_expired() or 0 + + skills = _prepare_graded_run(args) + gradable = _gradable_cases(args, skills) + + if not gradable: + print( + "[staleness] no evaluation in the selected skill(s) asserts " + "anything beyond routing, so there is nothing to compare. A " + "staleness run needs cases that say what the skill should do." + ) + return 0 + + # The runner's own `~/.claude/skills` are registered beside the staged + # ones, so without this the baseline is not an empty room -- it is this + # machine's skills minus the one under test. `routing` has refused to run + # without isolation for the same reason; staleness needs it more, because + # the baseline being empty is the entire measurement rather than one + # input to it. + if args.engine == "claude-code-no-sandbox" and not routing.can_isolate_config(): + raise SystemExit( + "error: staleness cannot run on --engine claude-code-no-sandbox " + f"without {' or '.join(routing.ENV_CREDENTIALS)}. The baseline " + "half has to be a room with no skills in it, and redirecting the " + "CLI away from the runner's own config dir only works when auth " + "comes from the environment. Without it every skill installed on " + "this machine joins the baseline, and a case it solves there " + "would be reported as one the skill under test was not needed " + "for.\n Set one of those, or use --engine claude-code, whose " + "guest has no user-level skills at all." + ) + + # One token for this invocation, stamped on every log both halves write. + # `--from-logs` has nothing else to pair on: the log directory is shared + # and never pruned, and a plain `behavioral` run stamps the same + # `BASELINE_KEY: False` the graded half does, so without this an unrelated + # old run is a candidate for this comparison's "with skill" side. + run_key = uuid.uuid4().hex + + passes = ( + "run twice" if args.attempts <= 1 + else f"run {args.attempts} time(s) on each side" + ) + print(f"[staleness] {len(gradable)} case(s), {passes}", flush=True) + # A directory per half, not one shared between them. The contamination + # check reads what the agent did, so it cannot see state the CLI itself + # writes into its config dir; were the graded half to leave anything + # there that the baseline half could pick up, the baseline would no + # longer be the empty room the whole command rests on. Nothing is known + # to do this, but a second temporary directory costs nothing and the + # question stops needing an answer. + with ( + tempfile.TemporaryDirectory(prefix="skillscope-staleness-with-") as graded_dir, + tempfile.TemporaryDirectory(prefix="skillscope-staleness-without-") as bare_dir, + ): + host = args.engine == "claude-code-no-sandbox" + with_skill = _run_behavioral( + args, skills, gradable, + config_dir=Path(graded_dir) if host else None, run_key=run_key, + ) + without = _run_behavioral( + args, skills, gradable, install=False, + config_dir=Path(bare_dir) if host else None, run_key=run_key, + ) + + comparisons = staleness.pair(with_skill, without) + summary = staleness.summarize( + comparisons, + { + "model": args.model, + "engine": args.engine, + "effort": args.effort, + "skills": skills, + "extended": args.extended, + "attempts": args.attempts, + # Recorded only when it bound. A `0` in the report would read as a + # cap of nothing rather than as no cap, and `render_markdown` + # warns about a partial run on the presence of this key. + **({"max_cases": args.max_cases} if args.max_cases else {}), + **({"concurrency": args.concurrency} if args.concurrency else {}), + "wall_time_s": round(time.time() - started, 1), + "timeout": args.timeout, + "github_run_id": os.environ.get("GITHUB_RUN_ID"), + **usage.snapshot().as_meta(), + **_sandbox_meta(args), + }, + ) + _write_report(summary, staleness.render_markdown(summary), args, "staleness") + if (code := _fail_if_expired()) is not None: + return code + return 0 + + # -------------------------------------------------------------------------- # Argument parsing # -------------------------------------------------------------------------- @@ -921,8 +1343,14 @@ def _add_routing_room_argument( ) -def _add_graded_arguments(parser: argparse.ArgumentParser) -> None: - """Flags shared by the routing and behavioral commands.""" +def _add_graded_arguments( + parser: argparse.ArgumentParser, *, timeout: float = deadline.DEFAULT_TIMEOUT_S +) -> None: + """Flags shared by the graded commands: routing, behavioral and staleness. + + `timeout` is threaded through rather than overridden afterwards so the + `--help` text matches the default in force. See `_add_timeout_argument`. + """ _add_skills_argument(parser) parser.add_argument( "--skill", @@ -997,18 +1425,28 @@ def _add_graded_arguments(parser: argparse.ArgumentParser) -> None: "comparable. Default: claude-code, or $SKILLSCOPE_HARNESS." ), ) - _add_timeout_argument(parser) + _add_timeout_argument(parser, timeout) -def _add_timeout_argument(parser: argparse.ArgumentParser) -> None: +def _add_timeout_argument( + parser: argparse.ArgumentParser, default: float = deadline.DEFAULT_TIMEOUT_S +) -> None: + """The whole-command deadline, with a help string that knows its default. + + `default` is a parameter rather than a `set_defaults` call afterwards + because the help text is built here and argparse does not revisit it. + `staleness` overrode the value and left the help saying 900, three lines + under a description announcing that it defaults to twice the usual -- a + screen of `--help` contradicting itself. + """ parser.add_argument( "--timeout", type=float, - default=deadline.DEFAULT_TIMEOUT_S, + default=default, help=( "Seconds before the whole command is abandoned. Shared by " - "structural, routing, and behavioral. Default: 900 (15 minutes). " - "0 disables." + "structural, routing, behavioral and staleness. " + f"Default: {default:g} ({default / 60:g} minutes). 0 disables." ), ) @@ -1193,6 +1631,125 @@ def build_parser() -> argparse.ArgumentParser: _add_graded_arguments(behavioral_parser) behavioral_parser.set_defaults(handler=cmd_behavioral) + staleness_parser = commands.add_parser( + "staleness", + help="Whether a skill's cases still need the skill.", + description=( + "Run every behavioral case twice -- once with the skill installed, " + "once with nothing installed -- and report which cases moved. A " + "case that passes either way is not evidence about the skill. " + "Reports only: a verdict here never fails a run, though an " + "expired deadline or a refused run still exits non-zero. It costs " + "twice what `behavioral` costs and wants more than twice the " + "--timeout, which is why --timeout defaults to twice the usual " + "here: both passes share one bound, and the baseline is the " + "slower half because an agent without the skill explores instead " + "of going straight to the work." + ), + ) + # Twice the shared bound, because the command is two passes under one. + # Passed into `_add_graded_arguments` rather than set afterwards, so the + # `--help` text is built from the number actually in force: a + # `set_defaults` override changes the value and leaves the help saying + # whatever it was built with. The measurement behind the number is in + # `cmd_staleness`: a two-case skill finished the graded pass in 2m23s and + # had not finished the baseline after 30 minutes, so the shared 900s gave + # the slower half ~637s of a measured 1800s+ need. + _add_graded_arguments(staleness_parser, timeout=2 * deadline.DEFAULT_TIMEOUT_S) + staleness_parser.add_argument( + "--attempts", + type=int, + default=1, + metavar="N", + help=( + "Run every case N times on each side instead of once. One pair of " + "runs is a lead rather than a result -- an agent disagrees with " + "itself between runs, so a case can pass with the skill and fail " + "without it for no reason the skill is responsible for. Repeats " + "are what let a reader weigh a verdict: the report prints how " + "many attempts passed on each side, and 3/3 against 0/3 is not " + "3/3 against 1/3. A side counts as having passed when any of its " + "attempts did, which makes a higher N a higher bar rather than a " + "steadier reading of the same one: earns_it at three attempts " + "needs the baseline to have failed all three times, and a " + "baseline that passes once takes the case to no_signal. So " + "verdicts compare across runs at equal N and not otherwise. " + "Costs N times what it costs now, on both sides. Default: 1." + ), + ) + staleness_parser.add_argument( + "--max-cases", + type=int, + default=0, + metavar="N", + help=( + "Grade only the first N cases of each selected skill, after " + "--only and --extended have chosen them. For trying the command " + "on a large suite without paying for all of it. Both sides run " + "the same N, and the report says the run did not cover the whole " + "suite. Default: 0, every case." + ), + ) + staleness_parser.add_argument( + "--concurrency", + type=int, + default=0, + metavar="N", + help=( + "How many cases may be in flight at once. This command's baseline " + "half is the slow one -- an agent with no skill explores -- and " + "running its cases one after another is where the wall clock " + "goes. Raise it as far as the engine's sandboxes and the model's " + "rate limits will take. Default: 0, inspect's own default." + ), + ) + staleness_parser.add_argument( + "--from-logs", + metavar="DIR", + help=( + "Rebuild the comparison from logs already on disk instead of " + "running anything. Both halves carry the skill, which side they " + "were, and the run that wrote them in their eval metadata, so a " + "log directory is enough to pair them -- and to tell this run's " + "halves from a neighbouring run of the same skill. Costs nothing " + "and reaches no model. Logs carrying no run id are passed over; " + "see --unlabelled-logs." + ), + ) + staleness_parser.add_argument( + "--unlabelled-logs", + action="store_true", + help=( + "With --from-logs, also read logs that carry no run id, which " + "means logs written before this command stamped one. Off by " + "default: an ordinary `behavioral` log is shaped exactly like a " + "pre-id graded half, so reading them unasked inflates the counts " + "with skills the run never covered, and can hand a comparison a " + "'with skill' half from a different run. Passing this asserts the " + "directory holds nothing else for these skills. A group still " + "needs a baseline side, and a side holding several logs is " + "reported `ungraded` rather than guessed at." + ), + ) + staleness_parser.add_argument( + "--run-id", + default="", + metavar="ID", + help=( + "With --from-logs, rebuild a named run rather than the most " + "recent. A log directory accumulates every staleness run done in " + "it, and a rebuild reads one per skill: reading all of them would " + "report the same case once per run and count it that many times " + "in the totals. The report names the run it read and lists the " + "ids it passed over, which is where this value comes from. An id " + "belongs to the invocation that minted it and covers whichever " + "skills that invocation was asked for, so naming one can leave a " + "skill in the directory unreported; when it does, the command " + "says which." + ), + ) + staleness_parser.set_defaults(handler=cmd_staleness) + select_parser = commands.add_parser( "select", help="Emit the CI plan for a change, as JSON.", diff --git a/skillscope/datasets.py b/skillscope/datasets.py index aea25f9..4464404 100644 --- a/skillscope/datasets.py +++ b/skillscope/datasets.py @@ -38,6 +38,12 @@ with a substring match is a worse version of a check this module already models as a field. +``staleness`` adds no third thing to write. It grades the behavioral cases +above a second time with the skill withheld, which makes it a check on this +dataset as much as on the skill: a case that passes either way is not +evidence about the skill, whether because the skill stopped mattering or +because the case was never evidence in the first place. + ``skill_should_trigger: false`` makes the evaluation routing-only. No skill loads for it, so there is no behavioral phase to hang an assertion or a staged workspace off, and those fields are rejected rather than silently ignored: diff --git a/skillscope/engine/behavioral.py b/skillscope/engine/behavioral.py index 06ca60f..b271d79 100644 --- a/skillscope/engine/behavioral.py +++ b/skillscope/engine/behavioral.py @@ -19,7 +19,7 @@ import sys from pathlib import Path -from .. import agent, config, deadline, routing as routing_core +from .. import agent, config, deadline, harness as harnesses, routing as routing_core from ..behavior import BehaviorOutcome from ..datasets import Case from . import convert, hooks, models, sandbox as sandbox_spec, scorers, stats @@ -85,6 +85,8 @@ def build_task( ctx: dict | None = None, *, solver_factory, + baseline: bool = False, + run_key: str | None = None, ): """One inspect `Task` per skill: its cases, its skill installed, its scorer. @@ -96,6 +98,11 @@ def build_task( Required, and keyword-only. It was optional while a built-in react agent was the default, and a caller that forgot it silently graded a different agent than it asked for. + + `baseline` only names the task. Whether the skill is actually staged is + the solver's business, so the flag and the factory have to agree -- the + name is what makes a disagreement visible in the log directory rather + than invisible in the numbers. """ from inspect_ai import Task @@ -110,7 +117,15 @@ def build_task( bound = deadline.active() return Task( - name=f"behavioral-{skill}", + name=f"behavioral-{skill}" + ("-baseline" if baseline else ""), + metadata={ + BASELINE_KEY: baseline, + SKILL_KEY: skill, + # Resolved here, where it is known to be right. See `SKILL_DIR_KEY`. + SKILL_DIR_KEY: str(Path(skill_dir).resolve()), + **({STALENESS_RUN_KEY: run_key} if run_key else {}), + }, + tags=["skillscope", "baseline" if baseline else "graded"], dataset=samples, setup=hooks.setup_solver(hook, skill, cases), solver=solver_factory(skill_dir), @@ -138,6 +153,235 @@ def _failed(skill: str, cases: list[Case], detail: str) -> list[BehaviorOutcome] ] +def _normalise(text: str) -> str: + """Lowercased with separators flattened, so one comparison covers both OSes.""" + return text.replace("\\", "/").lower() + + +# Said instead of a contamination reason when there is no way to look for one. +# Phrased as what is missing rather than as a finding: it is not a claim that +# the baseline was dirty, it is a refusal to claim it was clean. +UNKNOWN_ROOM = ( + "could not tell whether the baseline read the skill: the run did not " + "record where the skill was, and it is not on this machine" +) + + +def _skill_marks(skill: str, recorded: str | None = None) -> tuple[str, ...] | None: + """The paths that reading makes a room dirty, lowercased and separator-flat. + + Two of them. The skill's own directory, absolute, which is where an agent + that went looking would find it. And `/`, the staged copy + `no_sandbox.install_skill` writes under a workspace that changes per + sample, so only its tail is knowable here. The harness names that + directory -- `.claude/skills` for Claude Code, its own for anything else -- + so this asks it rather than spelling one harness's answer in, which would + read every other harness's staged copy as untouched. + + It asks the *default* harness, which is right while that is the only one + registered. Once a run can name another, the staged mark has to come off + the log beside `recorded`, for the reason `recorded` exists: a reader + standing on a different harness than the run is not evidence about the + run. + + `recorded` is `SKILL_DIR_KEY` off the log: where the skill was on the + machine that ran the agent. Preferred over resolving it here, because + resolving it here resolves it against *this* process. A reader standing + somewhere else is not evidence about where the agent stood. + + `None` when neither source can say where the skill was, which is an + unknown room rather than a clean one. Returning the staged mark alone + looked like an answer and was not: it is the subset of the check that + cannot fail, so every tool call that reached the skill by its real path + came back clean and the case was credited. See `SKILL_DIR_KEY`. + + The staged mark carries no leading separator on purpose. `no_sandbox` runs + the agent with `cwd=workspace`, which makes `cat .claude/skills/foo/...` + the natural shell form, and a mark anchored with a leading `/` missed + exactly that. `_mentions` supplies the boundary instead, which is what + keeps the mark from matching the middle of a longer name. + """ + home = recorded + if home is None: + try: + home = str(Path(config.active().skill_path(skill)).resolve()) + except (Exception, SystemExit): # noqa: BLE001 -- see below + # `skill_path` raises `SystemExit` rather than returning, so + # catching `Exception` alone let a rebuild of someone else's logs + # take the whole command down. + return None + staged = f"{harnesses.get().skills_dir}/{skill}" + return (_normalise(home), _normalise(staged)) + + +# Characters that continue a path component. A mark followed or preceded by +# one of these is part of a longer name, not the name itself. +_NAME_CHARS = "-_." + + +def _mentions(text: str, mark: str) -> bool: + """Whether `text` names `mark` as a whole path component. + + Plain `in` was the bug twice over. It is what made `/{skill}/` match + `/run/user/1000`, and it survived in the shell branch after the structured + branch was fixed, where it read `.claude/skills/foo-extra` as skill `foo`'s + own files. A sibling skill whose name extends this one's is not this one. + + So a hit has to end at a boundary -- end of string, or a character that + cannot continue a component -- and start at one. `foo` matches in + `/x/.claude/skills/foo/SKILL.md` and in `cat .claude/skills/foo/S`, and + does not match in `foo-extra`, `foo.bak` or `myfoo`. + """ + width = len(mark) + start = 0 + while (at := text.find(mark, start)) >= 0: + start = at + 1 + after = text[at + width : at + width + 1] + if after and (after.isalnum() or after in _NAME_CHARS): + continue + before = text[at - 1 : at] if at else "" + if before and (before.isalnum() or before in _NAME_CHARS): + continue + return True + return False + + +def _strings(value) -> list[str]: + """Every string anywhere in a tool call's arguments. + + Flat rather than by field name. The first version of this checked a fixed + list of path-ish argument names, which is a closed set against an open tool + surface: it had no entry for Glob's `pattern`, which is often the only + argument a Glob call carries, so that call was invisible. MCP tools name + their arguments whatever they like, so the list would have kept drifting. + + Checking every string is safe here only because the marks are boundary + matched and absolute. That is what removed the false positives, not the + field list. + """ + if isinstance(value, str): + return [value] + if isinstance(value, dict): + return [s for v in value.values() for s in _strings(v)] + if isinstance(value, (list, tuple)): + return [s for v in value for s in _strings(v)] + return [] + + +def _read_the_skill(call, marks: tuple[str, ...], skill: str) -> str | None: + """Whether this call reached into the skill's own directory. + + Every string argument is tested as written, and a `~`-relative one is + tested expanded as well. Expansion is skipped for anything with whitespace + in it: that is a command line rather than a path, and it is matched as + written. + + The graded side reads its own installed skill by design, so a hit there is + expected. It is the baseline where this means the room was not empty. + """ + arguments = getattr(call, "arguments", None) or {} + if not isinstance(arguments, dict): + return None + + reason = f"read the skill's own files ({skill})" + for value in _strings(arguments): + text = _normalise(value) + if any(_mentions(text, mark) for mark in marks): + return reason + # `~/x` names the same file by another spelling, and expanding it is + # context-free: `$HOME` is the agent's own, recorded in the + # environment rather than inferred from where this is running. + # + # `./x` and `../x` are deliberately *not* resolved. Resolving them + # needs the agent's cwd, which is `no_sandbox`'s per-sample workspace + # and appears nowhere in the tool-call record -- so `resolve()` here + # resolved them against skillscope's own cwd instead, and made the + # verdict depend on which directory the report was run from. It never + # could serve the case it was written for. + if value.startswith("~") and not any(c.isspace() for c in value): + try: + expanded = _normalise(str(Path(value).expanduser())) + except (Exception, SystemExit): # noqa: BLE001 -- not a path, then + continue + if any(_mentions(expanded, mark) for mark in marks): + return reason + + # The one mark that is not a path. `_bundled_skills` is the harness's own + # staging directory, so its name in any argument is the agent having found + # the staging area rather than a file of its own. + if any("_bundled_skills" in _normalise(v) for v in _strings(arguments)): + return reason + return None + + +def _inspect_room( + routing_engine, sample, skill: str, skill_dir: str | None = None +) -> tuple[str | None, str | None]: + """`(the skill that fired, why the room was not what it claimed)`. + + `skill_dir` is `SKILL_DIR_KEY` off the log, passed by a reader rebuilding + someone else's run. A live run leaves it `None` and resolves the skill + where it already knows it is. + + One walk over the tool calls, answering both questions, because the second + is only meaningful beside the first. `detect_activation` reports a skill + nobody staged as `other:`, which is exactly the contamination this + looks for -- the runner's own `~/.claude/skills` are registered next to + the staged ones, and a behavioral run passes no `config_dir` to keep them + out. + + `fired` is `None` or exactly `skill`, never an `other:` marker. The two + questions are answered by separate branches for that reason: when they + shared one `if/elif`, a second `other:` hit arriving after `why` was + already set fell through to the `elif` and was recorded as the skill + having fired -- which reads downstream as "the skill activated" on exactly + the contaminated cases that most need the opposite. + + Reading the skill's own source is the second route, and the one isolation + only partly closes: `no_sandbox` runs the agent with `cwd=workspace`, but + nothing stops it naming an absolute path, so a directory the staging step + skipped is still on disk for it to find. The sandboxed `claude-code` leg + does close it, by isolating the filesystem. A baseline that read it was + not a baseline. + + Best effort: a shape this cannot walk is an unknown room, not a failed + case. + """ + fired: str | None = None + why: str | None = None + marks = _skill_marks(skill, skill_dir) + # Nowhere to check against, so the read half of the check cannot run. Said + # out loud rather than passed over: this lands the case `ungraded`, which + # is what an unanswerable question is worth. Applied after the walk, not + # before it, so a reason the walk *could* establish -- another skill + # firing, which needs no path at all -- is reported in preference to it. + unknown, marks = marks is None, marks or () + try: + for call in routing_engine._tool_calls(sample): + hit = routing_core.detect_activation( + routing_engine._activation_event(call), + [skill], + allow_body_path=routing_engine.ALLOW_BODY_PATH, + ) + if hit and hit.startswith("other:"): + if why is None: + why = f"another skill fired: {hit.removeprefix('other:')}" + elif hit and fired is None: + fired = hit + if why is None: + why = _read_the_skill(call, marks, skill) + except Exception: # noqa: BLE001 -- see the docstring + # A walk cut short inspected only the calls it reached, so evidence on + # any later call was never looked at. That is an unknown room whatever + # `unknown` says: it was computed before the walk, from whether there + # were marks to check against, and cannot speak for a walk that did + # not finish. Returning the completed-walk expression here would + # report a baseline nobody inspected as clean, which is the one + # direction this check must never fail in. + return fired, why or UNKNOWN_ROOM + return fired, why or (UNKNOWN_ROOM if unknown else None) + + def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: """Map one inspect `EvalLog` back onto skillscope's outcome objects. @@ -153,6 +397,10 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: for the same reason; the two now agree. Cases with no sample are still errored below, so nothing goes quietly missing. """ + # Imported here rather than at module scope: `engine/routing.py` imports + # this module, so a top-level import would close the cycle. + from . import routing as routing_engine # noqa: PLC0415 + prompts = {c.id: c.prompt for c in cases} outcomes: list[BehaviorOutcome] = [] @@ -173,6 +421,8 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: elif not checks: error = "case has no behavioral assertions to grade" + activated, contaminated = _inspect_room(routing_engine, sample, skill) + outcomes.append( BehaviorOutcome( id=case_id, @@ -183,6 +433,11 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: checks=checks, error=error, degraded=routing_core.is_provider_error(error), + activated=activated, + contaminated=contaminated, + # Which repeat this is, where `--attempts` asked for several. + # 1 on every sample of an ordinary single-epoch run. + attempt=int(getattr(sample, "epoch", 1) or 1), ) ) @@ -207,6 +462,42 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: # stopped run reads back its own log and nobody else's. CALL_TOKEN = "skillscope_call" +# Which half of a staleness comparison a log is, and whose skill it is about. +# The task name says the same thing, but a name is a string somebody has to +# parse; inspect filters on metadata and tags, and so can anything reading a +# log directory afterwards. Written by both engines, so a reader does not have +# to know which one produced the log it is holding. +BASELINE_KEY = "skillscope_baseline" +SKILL_KEY = "skillscope_skill" + +# Which staleness invocation wrote this log. Only `cmd_staleness` sets it, and +# it mints one value per invocation, so two logs carrying the same value are +# the two halves of one comparison and nothing else is. +# +# `--from-logs` needs this and cannot be built without it. The log directory is +# shared by `behavioral`, `verify` and `routing`, is gitignored wholesale and +# is never pruned, and every ordinary `behavioral` run stamps +# `BASELINE_KEY: False` -- so grouping on `(skill, baseline)` alone splices an +# unrelated month-old run in as the "with skill" half, under a table that reads +# as confidently as a real one. `run_eval` reached the same conclusion for the +# same directory and solved it the same way, with `CALL_TOKEN`. +STALENESS_RUN_KEY = "skillscope_staleness_run" + +# Where the skill was on the machine that ran the agent. +# +# `_skill_marks` needs an absolute directory to recognise a tool call that +# reached into the skill. Deriving it at read time derives it from whoever is +# *reading*: a rebuild in another directory, or on a machine where the skill +# is not checked out, built a quietly weaker mark set and reported a baseline +# the live run had voided as `ungraded` back as `no_signal`. That is the +# "a rebuild is more permissive than the run it rebuilds" failure this module +# closes everywhere else; contamination was the one place it was still open. +# +# So the run records where the skill actually was, and the reader uses that +# rather than its own cwd. Written by both engines for the same reason +# `SKILL_KEY` is: a reader should not have to know which one produced the log. +SKILL_DIR_KEY = "skillscope_skill_dir" + def stop_requested() -> bool: """Whether the deadline's graceful stage has asked the command to stop.""" @@ -292,8 +583,31 @@ def run( *, solver_factory, log_dir: str | None = None, + baseline: bool = False, + run_key: str | None = None, + attempts: int = 1, + concurrency: int = 0, ) -> list[BehaviorOutcome]: - """Run every behavioral case, grouped by skill. Mirrors `behavior.run`.""" + """Run every behavioral case, grouped by skill. Mirrors `behavior.run`. + + `baseline` is passed through to the task and used in the progress lines. + It does not itself remove the skill: the caller's `solver_factory` does + that, and this says so in the output. + + `run_key` is passed through to the task's metadata. See + `STALENESS_RUN_KEY`: only a staleness run sets one, and it is what lets + `--from-logs` pair two halves without pairing two unrelated runs. + + `attempts` runs every case that many times, as inspect epochs, and the + epoch comes back on each sample as `BehaviorOutcome.attempt`. One attempt + is the ordinary run and leaves the logs exactly as they were. + + `concurrency` caps how many cases are in flight at once; 0 leaves + inspect's own default alone. Not raised by default: the agents these legs + drive hold real sandboxes and real rate limits, and a default that + multiplied either would be a change to every existing command rather than + an option on a slow one. + """ from inspect_ai import eval as inspect_eval sandbox_spec.require_provider() @@ -311,15 +625,31 @@ def run( outcomes.extend(_failed(skill, skill_cases, deadline.active().message())) continue - print(f"[behavioral] {skill}: {len(skill_cases)} case(s)", flush=True) + label = "baseline" if baseline else "behavioral" + each = "" if attempts <= 1 else f", {attempts} attempts each" + print(f"[{label}] {skill}: {len(skill_cases)} case(s){each}", flush=True) try: logs = run_eval( inspect_eval, - build_task(skill, skill_cases, model, solver_factory=solver_factory), + build_task( + skill, + skill_cases, + model, + solver_factory=solver_factory, + baseline=baseline, + run_key=run_key, + ), model=model, model_args=models.model_args(model), log_dir=log_dir or str(Path(".skillscope") / "logs"), log_realtime=realtime_logging(), + # Repeats are inspect's epochs: one task, one log, each case + # sampled `attempts` times with the epoch on every sample. + # Running the leg twice over would have written two logs a + # side, which is the shape `staleness._no_duplicates` refuses + # precisely because it cannot tell it from a collision. + epochs=attempts, + **({"max_samples": concurrency} if concurrency else {}), # A sample that raises is that case's failure, not the batch's. # inspect's default ends the task on the first error, so one # container that would not start discarded every case already diff --git a/skillscope/engine/no_sandbox.py b/skillscope/engine/no_sandbox.py index e614ede..b9d4cb0 100644 --- a/skillscope/engine/no_sandbox.py +++ b/skillscope/engine/no_sandbox.py @@ -417,6 +417,7 @@ def harness_solver( extra_flags: list[str] | None = None, stop_when_factory: Callable[[], Callable[[dict], str | None]] | None = None, room_names: list[str] | None = None, + install: bool = True, ): """Solver: install the skill, run the real CLI once, record what it did. @@ -453,6 +454,12 @@ def harness_solver( which skill fired, and everything after that is paid for and unread. Left `None`, this runs through `inspect_ai.util.subprocess` exactly as before. + `install` is what separates a graded run from its baseline. False stages + nothing, so the agent meets the prompt with no skill at all -- the failure + `install_skill` exists to prevent, asked for on purpose and labelled as + such by the caller. `skillscope staleness` runs both and compares them; + nothing else should pass False. + A factory rather than the rule itself because the solver is built once and run for every sample, while a budget counts *per case*. Handed a single rule, its tally would carry from one case into the next and the room would @@ -469,7 +476,8 @@ async def solve(state, generate): from inspect_ai.util import subprocess as sandbox_subprocess workspace = await _workspace() - install_skill(skill_dir, workspace, harness.skills_dir) + if install: + install_skill(skill_dir, workspace, harness.skills_dir) prompt = state.input_text spec = harnesses.RunSpec( @@ -548,6 +556,7 @@ def claude_code_no_sandbox( extra_flags: list[str] | None = None, stop_when_factory: Callable[[], Callable[[dict], str | None]] | None = None, room_names: list[str] | None = None, + install: bool = True, ): """`harness_solver` with Claude Code, the harness every caller has used.""" return harness_solver( @@ -559,4 +568,5 @@ def claude_code_no_sandbox( extra_flags, stop_when_factory, room_names, + install, ) diff --git a/skillscope/engine/verify.py b/skillscope/engine/verify.py index d06cde2..0061407 100644 --- a/skillscope/engine/verify.py +++ b/skillscope/engine/verify.py @@ -218,8 +218,17 @@ def build_task( model: str, ctx: dict | None = None, effort: str | None = None, + install: bool = True, + run_key: str | None = None, ): - """One task per skill, solved by real Claude Code rather than our agent.""" + """One task per skill, solved by real Claude Code rather than our agent. + + `install` False stages no skill, which is the baseline half of + `skillscope staleness`: the same cases, graded the same way, against an + agent that was never given the thing under test. The restore solver and + the agent's own skills argument have to agree about it, or the agent finds + a skill the report says it did not have. + """ from inspect_ai import Task from inspect_ai.solver import chain from inspect_swe import claude_code @@ -240,7 +249,21 @@ def build_task( bound = deadline.active() return Task( - name=f"claude-code-{skill}", + name=f"claude-code-{skill}" + ("" if install else "-baseline"), + # See `behavioral.build_task`: the same keys whichever engine ran. + metadata={ + behavioral.BASELINE_KEY: not install, + behavioral.SKILL_KEY: skill, + # See `behavioral.SKILL_DIR_KEY`: resolved on the machine that ran + # the agent, because a reader cannot work it out afterwards. + behavioral.SKILL_DIR_KEY: str(Path(skill_dir).resolve()), + # See `behavioral.STALENESS_RUN_KEY`: set only by a staleness run, + # and only there because only there are two logs meant to pair. + **( + {behavioral.STALENESS_RUN_KEY: run_key} if run_key else {} + ), + }, + tags=["skillscope", "graded" if install else "baseline"], setup=hooks.setup_solver(hook, skill, cases), cleanup=hooks.cleanup_fn(hook, skill, cases), dataset=samples, @@ -256,9 +279,11 @@ def build_task( # skill failed to do. solver=chain( _ensure_workdir(), - complete_skills_solver({skill_dir.name: skill_dir}), + complete_skills_solver({skill_dir.name: skill_dir} if install else {}), claude_code( - skills=[skill_dir], cwd=tools.workdir_path(), effort=effort or None + skills=[skill_dir] if install else [], + cwd=tools.workdir_path(), + effort=effort or None, ), ), scorer=scorers.expectations(), @@ -269,9 +294,27 @@ def build_task( def run( - skills: list[str], cases: list[Case], model: str, effort: str + skills: list[str], + cases: list[Case], + model: str, + effort: str, + *, + install: bool = True, + run_key: str | None = None, + attempts: int = 1, + concurrency: int = 0, ) -> list[BehaviorOutcome]: - """Mirrors `behavior.run`, so the CLI and the benchmark treat it the same.""" + """Mirrors `behavior.run`, so the CLI and the benchmark treat it the same. + + `run_key` is stamped on the task, same as the other engine's. See + `behavioral.STALENESS_RUN_KEY`. + + `attempts` and `concurrency` are inspect's epochs and sample limit, as in + `behavioral.run`. Both legs take them, so a staleness run measures the + same way whichever engine it asked for -- an option that worked on one + engine and was silently dropped on the other would make two runs of the + same command incomparable without saying so. + """ from inspect_ai import eval as inspect_eval require() @@ -291,15 +334,22 @@ def run( ) continue - print(f"[claude-code] {skill}: {len(skill_cases)} case(s)", flush=True) + label = "claude-code" if install else "claude-code/baseline" + each = "" if attempts <= 1 else f", {attempts} attempts each" + print(f"[{label}] {skill}: {len(skill_cases)} case(s){each}", flush=True) logs = behavioral.run_eval( inspect_eval, - build_task(skill, skill_cases, model, effort=effort), + build_task( + skill, skill_cases, model, effort=effort, + install=install, run_key=run_key, + ), model=model, model_args=models.model_args(model), log_dir=str(Path(".skillscope") / "logs"), log_realtime=behavioral.realtime_logging(), display="plain", + epochs=attempts, + **({"max_samples": concurrency} if concurrency else {}), ) if not logs and behavioral.stop_requested(): # See `behavioral.run`. diff --git a/skillscope/staleness.py b/skillscope/staleness.py new file mode 100644 index 0000000..08b40f2 --- /dev/null +++ b/skillscope/staleness.py @@ -0,0 +1,1314 @@ +# Copyright Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: MIT + +"""Is the skill earning its place, or would the agent have done as well alone? + +A behavioral run says whether a skill's cases pass. It cannot say whether they +pass *because of the skill*, and those are different questions: a case can be +written so that any competent agent satisfies it, at which point a green report +is measuring the model rather than the thing under test. + +So the same cases are run twice -- once with the skill installed and once with +nothing installed at all -- and the two are placed side by side. `install_skill` +calls the second situation out as a failure mode, "answering from the prompt and +scoring like it, which looks like a bad skill rather than a missing one". Asked +for deliberately and labelled, that failure is the measurement. + +Five things a case can turn out to be: + +``earns_it`` + Passes with the skill, fails without it, and the agent reached for the + skill. The case discriminates, which is the only outcome that tells you + the skill did something. +``unattributed`` + Passes with the skill and fails without it, but the agent never opened the + skill on an attempt that passed. Something moved; the skill is not shown + to be what moved it. Installing a skill does not make an agent use one, + and a difference that arrived without it is not evidence about it. +``no_signal`` + Passes either way. Says nothing about the skill, and usually says something + about the case: an expectation any agent meets is not evidence. +``harmful`` + Fails with the skill, passes without it. Rare, and worth stopping for: the + skill is actively steering the agent wrong. +``fails_either_way`` + Fails both times. A broken case or a genuinely hard one; either way the + comparison has nothing to report until it passes somewhere. + +Each side can be run more than once (``--attempts``). A side counts as having +passed when any of its attempts met every positive claim, which at one attempt +each is exactly the rule above. "Any" rather than "all" because the question a +side answers is what the agent *can* do: an agent that managed the case once +managed it, and a side that failed the other two times has still shown the +task to be within reach of whatever it was holding. + +That makes more attempts a higher bar rather than a steadier reading of the +same one. ``earns_it`` at three attempts needs the baseline to have failed +three times out of three, which is a stronger claim than failing once; and a +baseline that passes once in three takes the case to ``no_signal`` where one +attempt each would have called it ``earns_it``. So a verdict is comparable +with another taken at the same attempt count and not with one taken at a +different count, and raising ``--attempts`` is asking a harder question rather +than asking the same one more carefully. + +The counts are carried into the report either way, because `3/3` against `0/3` +and `1/1` against `0/1` are the same verdict and not the same evidence, and +`3/3` against `1/3` is a third thing again. + +No verdict here gates, though an expired deadline or a refused run still exits +non-zero: those are statements about the run rather than about a skill. A +nondeterministic agent disagrees with itself between runs often enough that a +single pair is evidence rather than proof -- the same reason +`tools/benchmark_engines.py` measures engine agreement against how much one +engine already disagrees with itself, and the same reason the `claude-code` leg +is documented as reporting rather than gating. A number here is a question to +go and look into, not a build failure. Repeats are what let a reader weigh one; +they are not a licence for this command to start failing builds. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field, replace + +from .behavior import BehaviorOutcome + +# (key, what it means) -- one row per verdict, in the order a reader should +# care about them. +VERDICTS: tuple[tuple[str, str], ...] = ( + ("earns_it", "passed with the skill, failed without it, and the skill fired"), + ("harmful", "failed with the skill, passed without it"), + ("unattributed", "passed only with the skill, but the skill never fired"), + ("no_signal", "passed either way"), + ("fails_either_way", "failed either way"), + ("guards_only", "asserts nothing the skill could be credited for"), + ("ungraded", "one side or both could not be graded"), +) + +# What a case claims the skill will *do*. Only these decide a verdict. +POSITIVE_KINDS = ("expected_behavior", "logs_contain", "files_exist") + +# What a case forbids. Deliberately excluded from the verdict. +# +# A guard rail exists to catch a future edit to the skill that starts doing the +# forbidden thing, so a well-written one holds whether or not the skill is +# installed -- the base model was never going to run `sudo` unprompted either. +# Counting that as "the skill made no difference" would mark the checks most +# worth keeping as dead weight, and push an author to delete them. They are +# reported on their own terms instead: held, or broken. +GUARD_KIND = "unexpected_behavior" + + +@dataclass +class Comparison: + """One case, run both ways, each way once or several times. + + No `diverged` flag is computed from the two sides' checks. The details are + judge prose, and two runs can word the same behaviour differently; a + boolean over that would claim a difference the harness cannot stand + behind. They are shown instead, for a reader to judge. + """ + + id: str + skill: str + verdict: str + # Which staleness run produced this comparison. Empty on a live run, + # where there is only one and the report is about it. A rebuild sets it, + # because a log directory holds every run anyone has ever done in it and + # a report that does not say which one it read is not a measurement of + # anything in particular. See `from_logs`. + run: str = "" + # Attempts that produced a grade, and how many of them met every positive + # claim. One attempt each is the default; `passes_with` is then 0 or 1 and + # the verdict rules read exactly as they did before repeats existed. + attempts_with: int = 0 + attempts_without: int = 0 + passes_with: int = 0 + passes_without: int = 0 + # Attempts that errored instead. Counted apart from the graded ones so a + # side that half ran is not reported as a side that half failed: the + # denominator a reader sees is the number of attempts that answered. + errored_with: int = 0 + errored_without: int = 0 + error: str | None = None + # The checks of the attempt the verdict rests on -- one that passed, where + # the side has one, and otherwise the first. Both sides are carried rather + # than reduced to the verdict. A case that failed both ways can still have + # failed them *differently* -- the first run of this command had an agent + # that tried the task and was stopped by the machine on one side, and one + # that never attempted it on the other, which is the skill working. The + # verdict could not see it and neither could the JSON, so the only way to + # find out was to open the raw inspect logs, which is not a report. + checks_with: list[dict] = field(default_factory=list) + checks_without: list[dict] = field(default_factory=list) + # Every guard rail the installed side was held to, one row per guard, + # failed if *any* attempt broke it. + # + # Kept apart from `checks_with` because that is one attempt's evidence -- + # `_decisive` picks it on positive claims and firing, knowing nothing + # about guards -- and a guard rail is a claim about the whole side. With + # `--attempts` above one, reading guards off the decisive attempt meant a + # skill that ran the forbidden thing on an attempt that was not chosen + # was reported as having broken nothing at all. + guards_with: list[dict] = field(default_factory=list) + # How much of the case each side satisfied, averaged over its graded + # attempts. The verdict is binary and these are not: a side passes only by + # meeting every positive claim, so `fails_either_way` covers a baseline + # that met four claims of five and one that met none, and `earns_it` + # covers a baseline that nearly managed it and one that never started. + # See `_claims`, which is where a reader meets them. + claims: int = 0 + claims_met_with: float = 0.0 + claims_met_without: float = 0.0 + # Whether the agent actually reached for the skill on the installed side, + # read off the attempt the verdict rests on -- which `_decisive` picks to + # be a passing attempt that fired, where the side has one. The verdict + # itself is decided over every passing attempt and not over this one; a + # case that would otherwise be `earns_it` and fired on no attempt that + # passed is `unattributed` instead. See `classify`. + activated_with: str | None = None + # Why the baseline was not the empty room it claims to be, if it was not. + # Reported whenever it happened, on any attempt. + baseline_contaminated: str | None = None + # Whether that happened on a baseline attempt that *passed*. This is the + # one that voids a verdict, and the distinction only has teeth once a side + # can be attempted more than once -- a baseline that was dirty on an + # attempt it failed cannot have reached a conclusion it never reached. + baseline_dirty_on_pass: bool = False + + +def positive(checks: list[dict]) -> list[dict]: + """The checks that claim the skill will do something.""" + return [c for c in checks if c.get("kind") in POSITIVE_KINDS] + + +def guards(checks: list[dict]) -> list[dict]: + """The checks that forbid something.""" + return [c for c in checks if c.get("kind") == GUARD_KIND] + + +def _met(outcome: BehaviorOutcome) -> bool: + """Whether this attempt met every positive claim on the case.""" + claims = positive(outcome.checks) + return bool(claims) and all(c["passed"] for c in claims) + + +def answered(attempts: list[BehaviorOutcome]) -> list[BehaviorOutcome]: + """The attempts that answered, as opposed to the ones that errored. + + An attempt that errored has measured nothing. Keeping it in the + denominator would report a provider that dropped the connection as a side + that failed, which is the most flattering possible reading of an error on + the graded side and the least flattering on the baseline. + """ + return [a for a in attempts if not a.error] + + +def _decisive(attempts: list[BehaviorOutcome]) -> BehaviorOutcome | None: + """The attempt a side's verdict rests on: one that passed, else the first. + + A side is credited with passing when *any* of its attempts did, so the + passing attempt is the one whose evidence belongs in the report. Where + nothing passed, every attempt is equally the answer and the first will do. + + Among passing attempts, one that reached for the skill is preferred over + one that did not. `classify` asks whether the skill fired on any attempt + that passed, and this is what keeps `activated_with` and `checks_with` + pointing at the same attempt as that answer: a report that said `earns_it` + while showing the checks of an attempt where nothing fired would be + describing a different run from the one it graded. + + A side where every attempt errored falls back to the first of those. Its + verdict is `ungraded` whatever this returns, and an errored attempt can + still carry the checks it had graded before it stopped -- which is the + difference between a report that shows how far the side got and one that + shows a blank. + """ + scored = answered(attempts) + for attempt in scored: + if _met(attempt) and attempt.activated: + return attempt + for attempt in scored: + if _met(attempt): + return attempt + if scored: + return scored[0] + return attempts[0] if attempts else None + + +def _worst_guards(attempts: list[BehaviorOutcome]) -> list[dict]: + """One row per guard rail this side was held to, at its worst showing. + + A guard rail that held twice and broke once broke: it exists to catch the + skill doing the forbidden thing, and it caught it. So the rows are keyed + on the expectation and a failure replaces a pass, which also keeps the + count at one per guard however many attempts ran. + + Errored attempts are left out, as everywhere else: an attempt that + measured nothing cannot have seen a guard hold either. + """ + worst: dict[str, dict] = {} + for attempt in answered(attempts): + for guard in guards(attempt.checks): + seen = worst.get(guard["expectation"]) + if seen is None or (seen["passed"] and not guard["passed"]): + worst[guard["expectation"]] = guard + return list(worst.values()) + + +def _claims_met(attempts: list[BehaviorOutcome]) -> float: + """Mean positive claims met per graded attempt.""" + scored = answered(attempts) + if not scored: + return 0.0 + met = sum(sum(1 for c in positive(a.checks) if c["passed"]) for a in scored) + return round(met / len(scored), 2) + + +def classify( + with_skill: list[BehaviorOutcome], without: list[BehaviorOutcome] +) -> str: + """Which of `VERDICTS` this case's two sides are. + + Both arguments are a side's attempts: one each is the ordinary run, and + `--attempts` makes them longer. A side passed when any attempt met every + positive claim -- see `GUARD_KIND` for why the forbidden ones are kept out + of it, and the module docstring for why "any" rather than "all". + + A side with nothing left to grade is `ungraded` rather than a failure. One + attempt that errored out of three is not that case: the other two answered. + """ + here_scored, there_scored = answered(with_skill), answered(without) + if not here_scored or not there_scored: + return "ungraded" + if not any(positive(a.checks) for a in here_scored + there_scored): + # Nothing was claimed, so nothing can be credited. Reporting this as + # `fails_either_way` would read as a broken case rather than as one + # that only ever pinned down what must not happen. + return "guards_only" + here = any(_met(a) for a in here_scored) + there = any(_met(a) for a in there_scored) + if here and not there: + # Installed is not used. The outcome differed, and the thing under + # test is not shown to be why -- so it is reported as its own finding + # rather than counted among the cases that needed the skill. + # + # Asked over the whole side rather than over one attempt. Reading + # `activated` off the attempt `_decisive` picked made the verdict turn + # on which of two passing attempts ran first: the same pair of runs in + # the other order credited the skill, and the order two attempts + # happened to run in is not a property of the skill. The module + # docstring's definition is "never opened the skill on an attempt that + # passed", and this is that sentence. + fired = any(_met(a) and a.activated for a in here_scored) + return _voided_if_dirty("earns_it" if fired else "unattributed", without) + if there and not here: + return _voided_if_dirty("harmful", without) + verdict = ( + "no_signal" if (here and there) + else "fails_either_way" + ) + return _voided_if_dirty(verdict, without) + + +# Verdicts a contaminated baseline cannot support, and why it cannot. +# +# Contamination is not symmetric. `no_signal` and `harmful` both rest on the +# baseline having *managed* something, and a baseline that reached another +# skill or read this skill's own files did not manage it unaided -- which is +# the whole of what those verdicts claim. +# +# `earns_it`, `unattributed` and `fails_either_way` rest on the baseline +# having *failed*. A baseline that failed while holding more than it should +# have fails all the harder without it, so the finding survives its own +# contamination and voiding it would throw away a conclusion the run is +# entitled to. +VOIDED_BY_CONTAMINATION = ("no_signal", "harmful") + + +def dirty_on_pass(without: list[BehaviorOutcome]) -> bool: + """Whether a baseline attempt that *passed* was contaminated. + + Contamination on an attempt the baseline failed voids nothing: the verdicts + it voids are the ones claiming the baseline managed something, and an + attempt that did not manage it cannot be the attempt they rest on. With one + attempt a side this is the same rule as before -- `no_signal` and `harmful` + both require the baseline to have passed -- and it only parts company with + it once a side can pass on one attempt and be dirty on another. + """ + return any(a.contaminated and _met(a) for a in answered(without)) + + +def _voided_if_dirty(verdict: str, without: list[BehaviorOutcome]) -> str: + if verdict in VOIDED_BY_CONTAMINATION and dirty_on_pass(without): + return "ungraded" + return verdict + + +def _by_case(outcomes: list[BehaviorOutcome]) -> dict[str, list[BehaviorOutcome]]: + """One side's attempts, grouped by the case each answers. + + A dict of lists rather than `{o.id: o}`: with `--attempts` above one a case + has several attempts on a side, and keying on the id alone kept whichever + came last and silently threw the rest away. + """ + grouped: dict[str, list[BehaviorOutcome]] = {} + for outcome in outcomes: + grouped.setdefault(outcome.id, []).append(outcome) + return grouped + + +def _why_dirty(attempts: list[BehaviorOutcome]) -> str | None: + """Why the baseline was not the empty room it claimed, over its attempts. + + A known reason beats `UNKNOWN_ROOM`. They are different findings -- one + says the baseline demonstrably held something, the other that nobody could + tell -- and a side with one attempt of each has the answer to the second + in the first, so reporting "could not be checked" would be throwing it + away. Errored attempts are read too: a run that stopped partway still saw + what it saw before it stopped. + """ + from .engine.behavioral import UNKNOWN_ROOM + + reasons = [a.contaminated for a in attempts if a.contaminated] + known = [r for r in reasons if r != UNKNOWN_ROOM] + return known[0] if known else (reasons[0] if reasons else None) + + +def _counts(attempts: list[BehaviorOutcome]) -> tuple[int, int, int]: + """`(graded, passed, errored)` for one side of one case.""" + scored = answered(attempts) + return ( + len(scored), + sum(1 for a in scored if _met(a)), + len(attempts) - len(scored), + ) + + +def pair( + with_skill: list[BehaviorOutcome], without: list[BehaviorOutcome] +) -> list[Comparison]: + """Match the two runs up by case id. + + A case missing from either side is reported as `ungraded` rather than + dropped. The two runs are separate evals and either can lose a case, so + quietly comparing the intersection would shrink the denominator and make + every remaining number look more confident than it is. + """ + with_by_id = _by_case(with_skill) + without_by_id = _by_case(without) + comparisons: list[Comparison] = [] + + for case_id in sorted(set(with_by_id) | set(without_by_id)): + these = with_by_id.get(case_id, []) + those = without_by_id.get(case_id, []) + here, there = _decisive(these), _decisive(those) + with_n, with_passes, with_errors = _counts(these) + without_n, without_passes, without_errors = _counts(those) + missing = ( + "the baseline run" if not those + else "the graded run" if not these + else None + ) + # The claims a case makes are the case's, not a side's, so either side + # can name them. The baseline is asked only when the graded attempt + # produced none -- a side that errored before it graded anything has + # no checks to count, and a denominator of 0 would print every case it + # lost as having claimed nothing. + # + # Tested on `here.checks` rather than on `here`. A `BehaviorOutcome` + # is a dataclass and so always truthy, which made `here or there` fall + # back only where the graded side was *absent* -- never in the case + # the comment above describes, which is the one it was written for. + naming = here if (here and here.checks) else (there or here) + claims = len(positive(naming.checks)) if naming else 0 + comparisons.append( + Comparison( + id=case_id, + skill=(these or those)[0].skill, + verdict="ungraded" if missing else classify(these, those), + attempts_with=with_n, + attempts_without=without_n, + passes_with=with_passes, + passes_without=without_passes, + errored_with=with_errors, + errored_without=without_errors, + error=( + f"no result from {missing}" if missing + else next( + (a.error for a in these + those if a.error), None + ) + ), + guards_with=_worst_guards(these), + checks_with=list(here.checks) if here else [], + checks_without=list(there.checks) if there else [], + claims=claims, + claims_met_with=_claims_met(these), + claims_met_without=_claims_met(those), + activated_with=here.activated if here else None, + baseline_contaminated=_why_dirty(those), + baseline_dirty_on_pass=dirty_on_pass(those), + ) + ) + return comparisons + + +def outcomes_from_log(log) -> tuple[str, bool, str | None, list[BehaviorOutcome]]: + """`(skill, was_baseline, run_key, outcomes)` for one inspect log. + + Rebuilt from the samples rather than from a dataset, because an offline + reader has no `Case` list: the log is all there is. Prompts are left empty + for the same reason -- the comparison is about verdicts, and inventing a + prompt field nobody wrote would be worse than an honest blank. + """ + from . import routing as routing_core + from .engine import behavioral, scorers + from .engine import routing as routing_engine + + eval_meta = log.eval.metadata or {} + skill = eval_meta.get(behavioral.SKILL_KEY) or log.eval.task + # Where the skill was when the agent ran, not where it is now. Absent from + # logs written before the key existed, which makes the room unknown rather + # than clean unless this machine happens to have the skill. See + # `behavioral.SKILL_DIR_KEY`. + skill_dir = eval_meta.get(behavioral.SKILL_DIR_KEY) + outcomes: list[BehaviorOutcome] = [] + for sample in log.samples or []: + checks: list[dict] = [] + for score in (sample.scores or {}).values(): + checks.extend((score.metadata or {}).get(scorers.CHECKS, [])) + error = sample.error.message if sample.error is not None else None + if error is None and not checks: + # Exactly what the live path sets (`behavioral._outcomes`), and it + # has to be the same string for the same reason the detector is + # shared: a sample whose checks could not be read rebuilds as + # `checks=[]`, which makes `_met()` False, which makes the case + # `earns_it` off the back of a harness failure. Without this the + # rebuild is *more permissive than the run it rebuilds*, quietly + # restoring a verdict the live run had voided as `ungraded`. + error = "case has no behavioral assertions to grade" + outcomes.append( + BehaviorOutcome( + id=str(sample.id), + skill=skill, + prompt="", + passed=error is None and bool(checks) and all(c["passed"] for c in checks), + elapsed_s=round(getattr(sample, "total_time", None) or 0.0, 2), + checks=checks, + error=error, + # inspect's own repeat counter. A run that asked for one + # attempt writes 1 on every sample, so this is the identity + # on everything but an `--attempts` run. + attempt=int(getattr(sample, "epoch", 1) or 1), + # Exactly what the live path sets, for the same reason as the + # error string above: a rebuild that left this False would + # report a provider outage as a skill that failed, which is + # the rebuild being more permissive than the run it rebuilds. + degraded=routing_core.is_provider_error(error), + # Both detected here too. Without the first, a rebuilt + # comparison reports every `earns_it` as never having fired; + # without the second it is *more permissive than the run it + # rebuilds*, quietly restoring verdicts the live run voided. + # + # `skill_dir` is what keeps the second true off this machine: + # the detector needs an absolute path to recognise, and left + # to work one out it works out this reader's rather than the + # run's. See `behavioral.SKILL_DIR_KEY`. + **dict( + zip( + ("activated", "contaminated"), + behavioral._inspect_room( + routing_engine, sample, skill, skill_dir + ), + ) + ), + ) + ) + return ( + skill, + bool(eval_meta.get(behavioral.BASELINE_KEY)), + eval_meta.get(behavioral.STALENESS_RUN_KEY), + outcomes, + ) + + +@dataclass +class Rebuild: + """The comparisons a log directory yielded, and the runs behind them. + + `runs` holds every staleness run the directory had for each skill, newest + first, whether or not it was used. Nothing prunes `.skillscope/logs`, so a + directory accumulates runs and "which one is this" stops being a detail: + the caller needs it to name the run in the report, and to say what a + `--run-id` could have asked for instead. + """ + + comparisons: list[Comparison] + runs: dict[str, list[str]] = field(default_factory=dict) + + +def from_logs( + log_dir: str, *, unlabelled: bool = False, run_id: str | None = None +) -> Rebuild: + """Rebuild the comparison from logs already on disk. + + Two runs become a pair because they carry the same skill, the same + `STALENESS_RUN_KEY`, and opposite `BASELINE_KEY` -- which is why all three + are written as metadata rather than inferred from a task name. + + Grouping on the run id is what makes this safe to point at + `.skillscope/logs`, which is the documented thing to do with it. That + directory is shared by `behavioral`, `verify` and `routing`, is gitignored + wholesale, and nothing prunes it; and every ordinary `behavioral` run + stamps `BASELINE_KEY: False`, so a run id is the only thing separating + this run's graded half from an unrelated run of the same skill last month. + + One run per skill, the newest, unless `run_id` names another. The run id + stops two runs being *spliced* into one comparison; it does not stop them + being *pooled*, and pooling is what a directory of accumulated runs + produces. Every run of a one-case skill used to add another copy of that + case to the report, so the second staleness run anybody did turned a + one-case suite into a two-case one and the headline counted it twice -- + under exactly the invocation the docs give for this. The runs passed over + are returned alongside, because a report that drops evidence silently is + the other half of the same fault. + + Newest is read off each log's own `created` stamp rather than the file's + mtime, which says when the directory was last copied about as often as it + says when the run happened. + + Logs with no run id are skipped unless `unlabelled` asks for them. See + `LEGACY_RUN` for what that costs and what it is for. They compete for + "newest" like any other run once asked for, and stay subject to the + ambiguity guard if they win. + + A directory holding only one side of a run is reported as a comparison + against nothing, the same way a case missing from one side is: silently + dropping it would read as a smaller, cleaner run. + + Logs that are not skillscope's own are passed over. A shared log directory + is normal, and failing on a neighbour's file would make this unusable in + exactly the directories it is meant for. + """ + from inspect_ai.log import list_eval_logs, read_eval_log + + from .engine import behavioral + + sides: dict[tuple[str, str], dict[bool, list[BehaviorOutcome]]] = {} + logs_per_side: dict[tuple[str, str], dict[bool, int]] = {} + written: dict[tuple[str, str], str] = {} + for info in list_eval_logs(log_dir): + log = read_eval_log(info.name) + if behavioral.BASELINE_KEY not in (log.eval.metadata or {}): + continue + skill, baseline, run_key, outcomes = outcomes_from_log(log) + key = (skill, run_key or LEGACY_RUN) + sides.setdefault(key, {}).setdefault(baseline, []).extend(outcomes) + tally = logs_per_side.setdefault(key, {}) + tally[baseline] = tally.get(baseline, 0) + 1 + written[key] = max(written.get(key, ""), str(log.eval.created or "")) + + candidates = [ + key + for key in sorted(sides) + if key[1] != LEGACY_RUN or (unlabelled and logs_per_side[key].get(True)) + ] + # Newest first, with a labelled run always ahead of the unlabelled + # bucket. `LEGACY_RUN` is a bucket and not a run -- it is every log that + # carried no id -- so letting it win on a timestamp would pick a pile over + # a measurement. The run id breaks the remaining ties, so two runs written + # in the same second order the same way on every machine. + runs: dict[str, list[str]] = {} + for key in sorted( + candidates, + key=lambda k: (k[1] != LEGACY_RUN, written.get(k, ""), k[1]), + reverse=True, + ): + runs.setdefault(key[0], []).append(key[1]) + chosen = [ + key + for key in candidates + if key[1] == (runs[key[0]][0] if run_id is None else run_id) + ] + + comparisons: list[Comparison] = [] + for key in sorted(chosen): + half = sides[key] + counts = logs_per_side[key] + ambiguous = key[1] == LEGACY_RUN and any(n > 1 for n in counts.values()) + comparisons.extend( + replace(comparison, run=key[1]) + for comparison in pair( + _no_duplicates(half.get(False, []), ambiguous), + _no_duplicates(half.get(True, []), ambiguous), + ) + ) + return Rebuild(comparisons=comparisons, runs=runs) + + +# The bucket for logs written before a run id was stamped on them. +# +# Skipped by default, and taken only when the caller asks with +# `--unlabelled-logs`. Both halves of that rule are load-bearing. +# +# Taking them by default was wrong, and not only for old directories. A plain +# `skillscope behavioral` run stamps `BASELINE_KEY: False` and never stamps a +# run id, because only `cmd_staleness` mints one -- so an ordinary behavioral +# log is shaped exactly like a pre-key graded half and always will be. Pooling +# them inflated `cases` and `ungraded` with every skill the user had ever run +# `behavioral` against, and could silently supply the "with skill" side of a +# comparison it was never part of. That is the splicing bug this module's run +# id exists to prevent, arriving through the back door. +# +# Refusing them outright was wrong too, and the reason outlives the migration +# that prompted it. A log directory is a record somebody already paid an agent +# run for; a reader that cannot open one because of a metadata key added after +# it was written has made that run worthless to settle a question it holds the +# answer to. The population is small and does not grow -- every run since the +# key arrived labels itself -- but "small" is the argument for a flag rather +# than for a refusal. +# +# So the caller asserts it. Even then a group needs a baseline side, which +# nothing but `cmd_staleness` ever writes, so a directory of ordinary +# `behavioral` logs stays out whatever flags it is given. And where a side +# holds more than one log there is a choice to make and no way to make it, so +# every case in that group is `ungraded` rather than guessed at. +# +# A labelled run never falls in here, so none of this can weaken a run written +# by this version. +LEGACY_RUN = "unlabelled" + + +def _no_duplicates( + outcomes: list[BehaviorOutcome], ambiguous: bool = False +) -> list[BehaviorOutcome]: + """`outcomes`, with every case this side cannot answer for errored. + + `ambiguous` errors all of them: see `LEGACY_RUN`, where unlabelled logs + leave several candidates for a side and no way to tell which run each + belongs to. Otherwise only a case this side answers twice *for the same + attempt* is errored. + + One run key and one side can still span several logs -- a run grades one + skill per task, and a run key covers the whole invocation -- and nothing + stops two of them carrying the same case id. `pair` groups a side's + attempts by case id, so without this two logs answering the same case + would read as one case attempted twice, and a collision between runs would + arrive disguised as the evidence `--attempts` exists to produce. + + The attempt number is what tells a repeat from a collision. A side that + ran three attempts answers each case three times by design, once per + attempt; two logs answering the same case *and* the same attempt is the + ambiguity, and no legitimate run produces one. + + Both copies are errored, not just the later one: which of them is this + side's answer is exactly what cannot be told, so keeping either would be + the guess. `classify` reads an errored attempt as one that measured + nothing, consistent with how this module reports every other ambiguity it + cannot resolve. + """ + if ambiguous: + return [ + replace( + outcome, + error="these logs carry no run id and there is more than one " + "of them on a side, so which run this case belongs to cannot " + "be told apart", + ) + for outcome in outcomes + ] + tally: dict[tuple[str, int], int] = {} + for outcome in outcomes: + key = (outcome.id, outcome.attempt) + tally[key] = tally.get(key, 0) + 1 + return [ + outcome if tally[(outcome.id, outcome.attempt)] == 1 + else replace( + outcome, + error="this run has more than one log reporting this case on the " + "same attempt, and which of them is its answer cannot be told " + "apart", + ) + for outcome in outcomes + ] + + +def summarize(comparisons: list[Comparison], meta: dict) -> dict: + from .engine.behavioral import UNKNOWN_ROOM + + counts = {key: 0 for key, _ in VERDICTS} + for comparison in comparisons: + counts[comparison.verdict] += 1 + + per_skill: dict[str, dict] = {} + for skill in sorted({c.skill for c in comparisons}): + subset = [c for c in comparisons if c.skill == skill] + per_skill[skill] = { + "cases": len(subset), + **{key: sum(1 for c in subset if c.verdict == key) for key, _ in VERDICTS}, + } + + graded = [ + c for c in comparisons if c.verdict not in ("ungraded", "guards_only") + ] + + # Guard rails, counted separately. A broken one is the serious finding + # here: a skill that made the agent do the forbidden thing. + broken = [ + c + for c in comparisons + if any(not g["passed"] for g in c.guards_with) + ] + guard_checks = sum(len(c.guards_with) for c in comparisons) + return { + "meta": meta, + "totals": { + "cases": len(comparisons), + "graded": len(graded), + **counts, + # Of the cases that were graded both ways, the share that moved + # when the skill was taken away *and* can be credited to the + # skill: `earns_it` alone. Cases that moved without the skill + # firing are `unattributed` and counted as that; the wider "moved" + # number is computed in `render_markdown`. This is the narrower + # one, and the one the run exists for. + "discriminating": ( + round(sum(1 for c in graded if c.verdict == "earns_it") / len(graded), 3) + if graded + else None + ), + "guard_checks": guard_checks, + "guards_broken_with_skill": len(broken), + # Counted apart, because they are different findings. One says + # the baseline was demonstrably not empty; the other says nobody + # could tell. Both void the same verdicts -- an unanswerable + # question is worth no more than a failed one -- but a report that + # totalled them together would claim contamination it never found. + "baseline_contaminated": sum( + 1 + for c in comparisons + if c.baseline_contaminated + and c.baseline_contaminated != UNKNOWN_ROOM + ), + "baseline_unknown": sum( + 1 + for c in comparisons + if c.baseline_contaminated == UNKNOWN_ROOM + ), + # Attempts that errored, over every case and both sides. A run + # that lost a third of its attempts to a flaky provider has + # measured a third less than its case count suggests, and nothing + # else in these totals says so. + "errored_attempts": sum( + c.errored_with + c.errored_without for c in comparisons + ), + }, + "per_skill": per_skill, + "cases": [asdict(c) for c in comparisons], + } + + +def _reasons(case: dict) -> list[tuple[str, str, str]]: + """Each expectation that failed on either side, with both sides' details. + + Matched on the expectation text, which is what both runs were graded + against. A check present on one side only still appears, so a run that + produced fewer checks than its partner cannot hide the difference. + """ + with_by_text = {c["expectation"]: c for c in case["checks_with"]} + without_by_text = {c["expectation"]: c for c in case["checks_without"]} + rows: list[tuple[str, str, str]] = [] + for text in sorted(set(with_by_text) | set(without_by_text)): + here, there = with_by_text.get(text), without_by_text.get(text) + if (here and here["passed"]) and (there and there["passed"]): + continue + rows.append( + ( + text, + _detail(here), + _detail(there), + ) + ) + return rows + + +def _unsettled(case: dict) -> bool: + """Whether either side passed some of its attempts and failed others. + + It matters most for `no_signal`, whose advice is to stop treating the case + as evidence about the skill. That is right when both sides managed it + every time and wrong when a side managed it once in three: a baseline that + passed once is enough to deny the case `earns_it` -- see the module + docstring on what "any attempt passed" costs -- and nowhere near enough to + call the case dead. Telling an author to delete it on that footing is the + user-facing harm in the rule, and this is what separates the two. + + Always False at one attempt a side, where a side either passed or did not. + """ + return any( + attempts and 0 < passes < attempts + for passes, attempts in ( + (case["passes_with"], case["attempts_with"]), + (case["passes_without"], case["attempts_without"]), + ) + ) + + +def _passes(case: dict) -> str: + """`with / without` as attempts passed out of attempts graded. + + The verdict is what the two sides did; this is how often they did it. At + one attempt a side it reads `1/1 with, 0/1 without` and carries no more + than the verdict already said. Above one attempt it is the whole point, + because the verdict cannot show it: `3/3` against `0/3` is `earns_it` on + a baseline that never managed it, `3/3` against `1/3` is `no_signal` on a + baseline that managed it once, and `2/3` against `1/3` is `no_signal` on + two sides that both disagreed with themselves. Only the first is a result, + and only these counts say which one a reader is holding. + """ + lost = case["errored_with"] + case["errored_without"] + tail = f" ({lost} attempt(s) errored)" if lost else "" + return ( + f"{case['passes_with']}/{case['attempts_with']} with, " + f"{case['passes_without']}/{case['attempts_without']} without{tail}" + ) + + +def _claims(case: dict) -> str: + """How much of the case each side met, averaged over its graded attempts. + + A side "passes" only by meeting every positive claim, so the verdict is + deaf to how close the other side came. `fails_either_way` covers a + baseline that met four of five claims and one that met none; `earns_it` + covers a baseline that nearly managed it and one that never started. Those + are different findings about the case, and an author deciding whether a + case still tests anything needs the one the verdict threw away. + + Above one attempt a side it also shows what the pass counts cannot: a mean + of 4.5 of 5 is a side that met everything once and nearly everything the + next time. + """ + if not case["claims"]: + return "-" + return ( + f"{case['claims_met_with']:g}/{case['claims']} with, " + f"{case['claims_met_without']:g}/{case['claims']} without" + ) + + +def _detail(check: dict | None) -> str: + if check is None: + return "not graded this side" + if check["passed"]: + return "passed" + return (check["detail"] or "failed").replace("|", "\\|").replace("\n", " ") + + +def render_markdown(summary: dict) -> str: + totals = summary["totals"] + meta = summary["meta"] + share = totals["discriminating"] + headline = ( + f"**{totals['earns_it']}/{totals['graded']} graded cases needed the skill**" + if totals["graded"] + else "**Nothing was graded both ways**" + ) + + # Said only when it is known. A `--from-logs` rebuild runs no model, and + # the invocation's own `--model`/`--effort` defaults describe the command + # that read the logs rather than the runs that wrote them -- rebuilding a + # `sonnet`/`low` run with a bare `--from-logs` printed "on `opus` (effort + # `high`)", which is not a hedge but a false statement about a measurement. + ran_on = ( + f" on `{meta['model']}` (effort `{meta['effort']}`)" + if meta.get("model") and meta.get("effort") + else "" + ) + # How many times each side was attempted, said plainly: `3/3` against + # `0/3` and `1/1` against `0/1` are the same verdict and not the same + # evidence, and a reader who does not know which they are holding cannot + # tell them apart. Read off the cases rather than the invocation's own + # `--attempts`, so a `--from-logs` rebuild describes the runs it read. + per_side = {c["attempts_with"] for c in summary["cases"]} | { + c["attempts_without"] for c in summary["cases"] + } + most = max(per_side - {0}, default=1) + # Zero is kept here where `most` discards it. A side with no attempts at + # all is a case that ran on one side only -- `pair`'s "missing from either + # side", or a deadline that landed mid-run -- and the sentence below used + # to say every case ran twice straight over the top of it. + least = min(per_side, default=1) + sides = ( + "twice: once with the skill installed, once with nothing installed" + if most <= 1 + else f"up to {most} times on each side, with the skill installed and " + "with nothing installed" + ) + ran = ( + f"Some cases ran on one side only; the rest ran {sides}." + if least == 0 + else f"Every case ran {sides}." + ) + lines = [ + "## Skill staleness", + "", + f"{headline}{ran_on}. {ran}", + "", + ] + if most > 1: + # Said where it cannot be missed, because the verdict's meaning moves + # with the attempt count and nothing else in the report says so. "Any + # attempt passed" makes a higher count a higher bar: see the module + # docstring. + lines += [ + "> **Verdicts from this run compare with other runs at " + f"{most} attempt(s), not with runs at a different count.** A side " + "counts as having passed when any of its attempts did, so more " + "attempts is a harder question: `earns_it` needs the baseline to " + "have failed every single time.", + "", + ] + if meta.get("max_cases"): + # A capped run is not a statement about the suite. Said in the one + # place a reader cannot miss it, because a partial pass that reads as + # a whole one is the failure a cap invites. + lines += [ + f"> **This run covered only the first {meta['max_cases']} case(s) " + "of each selected skill** (`--max-cases`). It says nothing about " + "the cases it did not run.", + "", + ] + if meta.get("from_logs"): + lines += [ + f"Rebuilt offline from the logs in `{meta['from_logs']}`; nothing " + "was run. The model and effort are whatever wrote them, which the " + "logs do not have to agree on, so they are not claimed here.", + "", + ] + # Which run this is a report about. A log directory accumulates every + # staleness run anybody has done in it, so naming the one read is the + # difference between a measurement and a pile of them -- see + # `from_logs` on what reading the pile used to do to the totals. + used = {c["skill"]: c["run"] for c in summary["cases"] if c["run"]} + if used: + lines += [ + "Read from " + + ", ".join( + f"run `{run}` for `{skill}`" + for skill, run in sorted(used.items()) + ) + + ".", + "", + ] + # A run id belongs to an invocation and not to a skill: a staleness + # run mints one id and stamps it on every skill it covers. So "was + # this run read" is a question about the whole report rather than + # about one skill's row -- asked per skill, a run read for `beta` was + # announced as unread by `alpha`, two lines under the sentence naming + # it as read. + # + # Listed per skill even so, because that is the only form in which + # the ids are actionable: `--run-id` is global, and which skills an + # id covers is whichever its own invocation was asked for. + read = set(used.values()) + unread = { + skill: [run for run in ids if run not in read] + for skill, ids in (meta.get("runs") or {}).items() + } + unread = {skill: ids for skill, ids in unread.items() if ids} + spare = {run for ids in unread.values() for run in ids} + if used and spare: + shown = "; ".join( + ", ".join(f"`{run}`" for run in ids[:3]) + + (f" and {len(ids) - 3} more" if len(ids) > 3 else "") + + f" for `{skill}`" + for skill, ids in sorted(unread.items()) + ) + lines += [ + f"> **{len(spare)} other staleness run(s) in this directory " + "were not read.** One run per skill is read" + + ("" if meta.get("run_id") else ", the most recent") + + ", because reading several reports the same case once per " + f"run: {shown}. `--run-id` reads one of them instead. An id " + "covers whichever skills its own run was asked for, which " + "need not be every skill here.", + "", + ] + + # Nothing moved at all. The row `earns_it | 0` is true and says nothing, + # and this is the one result the command exists to surface, so it is said + # rather than left to be noticed in a table. + # + # `unattributed` and `harmful` are checked too, and not for tidiness: a run + # whose only case was `unattributed` printed this, and its second sentence + # -- "every graded case came out the same with the skill and without it" -- + # was false of the only case in it. That case passed with the skill and + # failed without it; what it lacked was evidence the skill was why. Those + # two findings have their own notes, which say the true thing. + moved = totals["earns_it"] + totals["unattributed"] + totals["harmful"] + if totals["graded"] and not moved: + lines += [ + "> **No case in this suite shows the skill doing anything.** Every " + "graded case came out the same with the skill and without it. " + "Either the skill has stopped earning its place, or its cases " + "never tested what it knows -- read the skill and ask whether any " + "case here would need it. Those are different problems with " + "different fixes.", + "", + ] + + lines += [ + "Reporting only. Agents disagree with themselves between runs, so treat " + "a surprising verdict as somewhere to look rather than as a result.", + "", + "| Verdict | Cases | Means |", + "| --- | --- | --- |", + ] + for key, meaning in VERDICTS: + lines.append(f"| `{key}` | {totals[key]} | {meaning} |") + + if share is not None: + lines += ["", f"Discriminating: {share:.0%} of the cases graded both ways."] + + if totals["errored_attempts"]: + # Said out loud rather than left in the JSON. An attempt that errored + # is one nobody paid for an answer from, and a reader comparing `2/2` + # against `3/3` on the next case has no way to know the first was + # asked the same question three times. + lines += [ + "", + f"{totals['errored_attempts']} attempt(s) errored and are out of " + "the counts below. A case whose every attempt errored on a side " + "is `ungraded`; one that lost some of them is graded on the rest, " + "over the smaller denominator the counts show.", + ] + + from .engine.behavioral import UNKNOWN_ROOM + + unknown = [ + c + for c in summary["cases"] + if c["baseline_contaminated"] == UNKNOWN_ROOM + ] + if unknown: + lines += [ + "", + f"> **{len(unknown)} case(s) could not be checked for a clean " + "baseline.** The run did not record where the skill was, and it " + "is not on this machine, so there is nothing to recognise a " + "tool call against. They are reported `ungraded` where the " + "verdict rested on the baseline having managed something. " + "Re-running them records it.", + ] + + dirty = [ + c + for c in summary["cases"] + if c["baseline_contaminated"] and c["baseline_contaminated"] != UNKNOWN_ROOM + ] + if dirty: + voided = [c for c in dirty if c["verdict"] == "ungraded"] + # Split four ways, not two, and each group worded for what it actually + # rests on. + # + # A `guards_only` case makes no positive claim at all, so it does not + # rest on the baseline having failed and saying that it does is a + # claim the run cannot support -- the same reading of its own + # contamination rule that `classify` gets right by returning before + # the rule is consulted, and that this paragraph used to get wrong by + # filtering on contamination alone. + # + # The fourth group is what `--attempts` opened up. `_why_dirty` reads + # every attempt and `dirty_on_pass` reads only the ones that passed, + # so a baseline that was dirty on an attempt it failed and clean on + # the one it passed keeps its verdict. Where that verdict is one of + # `VOIDED_BY_CONTAMINATION` it rests on the baseline having *passed*, + # so the sentence about resting on a failure is false for it. + # + # Driven off `VOIDED_BY_CONTAMINATION` rather than a verdict list + # written out here, so the two cannot drift apart. + silent = [c for c in dirty if c["verdict"] == "guards_only"] + standing = [ + c for c in dirty + if c["verdict"] not in ("ungraded", "guards_only") + ] + stood = [ + c for c in standing + if c["verdict"] not in VOIDED_BY_CONTAMINATION + ] + elsewhere = [ + c for c in standing if c["verdict"] in VOIDED_BY_CONTAMINATION + ] + reasons = sorted({c["baseline_contaminated"] for c in dirty}) + note = [ + "", + f"> **{len(dirty)} case(s) had a baseline that was not empty**: " + + "; ".join(reasons) + + ".", + ] + if voided: + note.append( + f"> {len(voided)} of them are reported `ungraded`: a baseline " + "that reached another skill, or read this skill's own files, " + "cannot show what an agent manages unaided.", + ) + if stood: + note.append( + f"> {len(stood)} still stand -- " + + ", ".join(f"`{c['id']}` ({c['verdict']})" for c in stood) + + " -- because they rest on the baseline *failing*, and it " + "failed while holding more than it should have.", + ) + if elsewhere: + note.append( + f"> {len(elsewhere)} case(s) rest on a clean attempt -- " + + ", ".join( + f"`{c['id']}` ({c['verdict']})" for c in elsewhere + ) + + " -- the baseline passed on an attempt that was clean and " + "was dirty on one it failed, so the attempt the verdict rests " + "on is not the contaminated one. They stand, on weaker " + "evidence than a run with no contamination in it at all.", + ) + if silent: + note.append( + f"> {len(silent)} are `guards_only` -- " + + ", ".join(f"`{c['id']}`" for c in silent) + + " -- and claim nothing either way, so a dirty baseline " + "neither voids them nor leaves them standing on anything.", + ) + lines += note + + unfired = totals["unattributed"] + if unfired: + lines += [ + "", + f"> **{unfired} case(s) passed only with the skill installed, but " + "the skill never fired on the attempt that passed.** Installing a " + "skill does not make an agent use it, so for those the outcome " + "differed without the skill being shown to be the reason. They " + "are `unattributed` rather than `earns_it`: a lead to go and read, " + "not a case the skill has been credited with.", + ] + + if totals["guard_checks"]: + broken = totals["guards_broken_with_skill"] + lines += [ + "", + f"{totals['guard_checks']} guard rail(s) checked and kept out of " + "the verdict: they hold whether or not the skill is installed, " + "which is what they are for. " + + ( + f"**{broken} case(s) broke one with the skill installed** -- " + "that is the skill doing the forbidden thing." + if broken + else "None was broken with the skill installed." + ), + ] + + unsettled = [ + c for c in summary["cases"] + if c["verdict"] == "no_signal" and _unsettled(c) + ] + if unsettled: + lines += [ + "", + f"> **{len(unsettled)} case(s) are `no_signal` because a side " + "disagreed with itself**, not because both sides managed it every " + "time: " + + ", ".join(f"`{c['id']}`" for c in unsettled) + + ". One baseline attempt in three is enough to stop a case " + "being `earns_it` and is not enough to call it dead. Run them " + "again before acting on them.", + ] + + notable = [ + c for c in summary["cases"] + if c["verdict"] in ("no_signal", "harmful", "unattributed") + ] + lines += ["", "### Cases worth a look", ""] + if not totals["graded"] and not totals["guard_checks"]: + # Not the same sentence as "none of them was worth a look". A run with + # nothing graded has no graded cases to make either claim about, and + # saying every one of them came out well is the same bug commit + # `542cad9` fixed one aisle over. + lines.append( + "No case was graded both ways, so there is nothing to look at here." + ) + elif not totals["graded"]: + # A `guards_only` case is not graded and is not nothing: it ran on + # both sides and its guards were checked. "Nothing to look at here", + # printed two paragraphs under "that is the skill doing the forbidden + # thing", is the strongest finding this command can produce being + # contradicted by the weakest sentence in it. This section is about + # the verdict table, so it says that instead. + lines.append( + "No case reached a verdict. The guard rails above are what this " + "run found." + ) + elif not notable: + lines.append( + "None. Every graded case either needed the skill or failed both ways." + ) + else: + lines += [ + "| Case | Verdict | Passed | Claims met | What to do |", + "| --- | --- | --- | --- | --- |", + ] + for case in notable: + advice = ( + "the two sides did not settle; run it again before reading " + "anything into it" + if case["verdict"] == "no_signal" and _unsettled(case) + else { + "harmful": "the skill steered the agent wrong here", + "unattributed": "something moved, but the agent never " + "opened the skill -- read the run before crediting it", + }.get( + case["verdict"], + "the agent already did this unaided; the case is not " + "evidence about the skill", + ) + ) + lines.append( + f"| `{case['id']}` | {case['verdict']} | {_passes(case)} " + f"| {_claims(case)} | {advice} |" + ) + + # Every claim that came out differently on the two sides -- whatever the + # verdict. This used to skip `earns_it` cases, on the reasoning that a + # skill which earned its keep needs no explaining, which had it exactly + # backwards: those are the only cases where the evidence lives, and the + # section printed "nothing to show" on a run whose whole point was that + # three cases moved. + detailed = [c for c in summary["cases"] if _reasons(c)] + lines += ["", "### What the skill changed", ""] + if not detailed: + # Two findings that used to share one sentence. "Every claim matched" + # is a statement about the skill; "no case was graded on both sides" + # is a statement about the run, and printing the first for the second + # tells a reader the run concluded something it never reached. + compared = any( + c["checks_with"] and c["checks_without"] for c in summary["cases"] + ) + lines.append( + "Nothing: every claim came out the same with the skill and " + "without it." + if compared + else "Nothing was compared: no case was graded on both sides." + ) + for case in detailed: + lines += [ + f"**`{case['id']}`** ({case['verdict']}) -- {_passes(case)}; " + f"claims met {_claims(case)}", + "", + "| Expectation | With the skill | Without it |", + "| --- | --- | --- |", + ] + for text, here, there in _reasons(case): + lines.append( + f"| {text[:70].replace('|', chr(92) + '|')} " + f"| {here[:90]} | {there[:90]} |" + ) + lines.append("") + return "\n".join(lines) + "\n" diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 8fb988d..06ca6a9 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -34,6 +34,7 @@ import time import types import unittest +from dataclasses import asdict import urllib.error from pathlib import Path from unittest import mock @@ -50,6 +51,7 @@ engine as engine_module, references, routing, + staleness, structure, ) from skillscope import selection as select_module @@ -2391,7 +2393,12 @@ def test_the_dispatch_reads_that_set_rather_than_a_literal(self) -> None: # The bug was a branch list that fell behind the choices list, so # the source is what to assert: every engine named, and an `else` # that raises rather than quietly picking one. - source = inspect.getsource(cli.cmd_behavioral) + # + # Read from `_run_behavioral` rather than `cmd_behavioral`: the + # dispatch moved there when `staleness` began running the same cases + # with the skill withheld, so both commands reach an engine by one + # route. The property being guarded is unchanged. + source = inspect.getsource(cli._run_behavioral) for engine in cli.ENGINES: with self.subTest(engine=engine): self.assertIn(f'args.engine == "{engine}"', source) @@ -5980,5 +5987,2178 @@ def test_a_good_table_still_binds(self) -> None: self.assertTrue(engine_models.cost_limit_binds("mockllm/model")) +def _outcome( + case_id: str, + passed: bool, + *, + error: str | None = None, + skill: str = "s", + activated: str | None = "s", + attempt: int = 1, +): + """One behavioral outcome, as either half of a staleness pair would report it. + + Carries a positive claim matching `passed`, because that is what a verdict + is computed from -- an outcome asserting nothing is `guards_only`, which + has its own tests. + + `activated` defaults to the skill having fired, which is the ordinary run + and keeps every test that is not about activation about what it is about. + The tests that *are* about it pass `activated=None` -- see + `TestEarningItMeansTheSkillActuallyFired`, where a case that moved without + the skill firing is `unattributed` rather than `earns_it`. + """ + return behavior.BehaviorOutcome( + id=case_id, skill=skill, prompt="p", passed=passed, elapsed_s=0.0, error=error, + activated=activated, + attempt=attempt, + checks=[{ + "kind": "expected_behavior", "expectation": "do the thing", + "passed": passed, "detail": "", + }], + ) + + +class TestStalenessSeparatesTheSkillFromTheModel(unittest.TestCase): + """The four verdicts, and why each is not the others.""" + + def test_a_case_that_needs_the_skill_is_the_one_that_moved(self) -> None: + self.assertEqual( + staleness.classify([_outcome("a", True)], [_outcome("a", False)]), "earns_it" + ) + + def test_a_case_that_passes_unaided_is_not_evidence(self) -> None: + # The finding the command exists for: green with the skill, green + # without it, so the report was measuring the model. + self.assertEqual( + staleness.classify([_outcome("a", True)], [_outcome("a", True)]), "no_signal" + ) + + def test_a_skill_that_makes_things_worse_is_called_that(self) -> None: + self.assertEqual( + staleness.classify([_outcome("a", False)], [_outcome("a", True)]), "harmful" + ) + + def test_a_case_failing_both_ways_says_nothing_either(self) -> None: + self.assertEqual( + staleness.classify([_outcome("a", False)], [_outcome("a", False)]), + "fails_either_way", + ) + + def test_an_errored_side_is_ungraded_rather_than_no_difference(self) -> None: + # A dropped connection measured nothing. Reading it as "the skill made + # no difference" is the most flattering possible reading of an error. + self.assertEqual( + staleness.classify( + [_outcome("a", False, error="API Error: 504")], [_outcome("a", False)] + ), + "ungraded", + ) + + +class TestStalenessPairsEveryCaseItWasAskedFor(unittest.TestCase): + def test_a_case_missing_from_one_side_is_kept_as_ungraded(self) -> None: + # Comparing the intersection would shrink the denominator and make + # every surviving number look more confident than it is. + comparisons = staleness.pair([_outcome("a", True), _outcome("b", True)], [_outcome("a", False)]) + verdicts = {c.id: c.verdict for c in comparisons} + self.assertEqual(verdicts, {"a": "earns_it", "b": "ungraded"}) + missing = next(c for c in comparisons if c.id == "b") + self.assertIn("baseline", missing.error) + + def test_the_share_is_over_what_was_graded_both_ways(self) -> None: + comparisons = staleness.pair( + [_outcome("a", True), _outcome("b", True), _outcome("c", True)], + [_outcome("a", False), _outcome("b", True)], + ) + totals = staleness.summarize(comparisons, {"model": "m", "effort": "e"})["totals"] + self.assertEqual(totals["cases"], 3) + self.assertEqual(totals["graded"], 2) + self.assertEqual(totals["discriminating"], 0.5) + + def test_nothing_graded_leaves_the_share_unset_rather_than_zero(self) -> None: + # Zero would read as "the skill never mattered", which is a result. + # None is the absence of one. + comparisons = staleness.pair([_outcome("a", True)], []) + totals = staleness.summarize(comparisons, {"model": "m", "effort": "e"})["totals"] + self.assertIsNone(totals["discriminating"]) + + +class TestTheStalenessReportLeadsWithTheCaveat(unittest.TestCase): + def test_the_reader_is_told_it_does_not_gate_before_the_numbers(self) -> None: + comparisons = staleness.pair([_outcome("a", True)], [_outcome("a", True)]) + summary = staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + report = staleness.render_markdown(summary) + self.assertIn("Reporting only", report) + self.assertLess(report.index("Reporting only"), report.index("| Verdict |")) + + def test_a_case_that_needed_nothing_is_named_with_what_to_do(self) -> None: + comparisons = staleness.pair([_outcome("a", True)], [_outcome("a", True)]) + report = staleness.render_markdown( + staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + ) + self.assertIn("`a`", report) + self.assertIn("unaided", report) + + +@contextlib.contextmanager +def _staleness_harness(test, *, dataset=None): + """A repo `cmd_staleness` can be run against, with the two halves recorded. + + The command was never called by a test before this: every assertion about + it read its source. A regression that called `_run_behavioral` twice with + the same `install` -- which destroys the entire measurement, since the two + halves become the same run -- passed the whole suite. + + `_run_behavioral` is the seam, so what each half was asked for is the + thing recorded. Everything below it reaches a provider. + """ + repo = Repo(test) + repo.skill("alpha", dataset=dataset or _behavioral_dataset("alpha")) + repo.activate() + + calls: list[dict] = [] + + def record(args, skills, gradable, *, install=True, config_dir=None, run_key=None): + calls.append( + { + "install": install, + "config_dir": config_dir, + "run_key": run_key, + "cases": [c.id for c in gradable], + } + ) + return [] + + harness = types.SimpleNamespace(repo=repo, calls=calls) + + def run(engine="claude-code", extra=None): + argv = [ + "staleness", "--engine", engine, "--skip-preflight", + "--model", "mockllm/model", *(extra or []), + ] + args = cli.build_parser().parse_args(argv) + return cli.cmd_staleness(args) + + harness.run = run + + with mock.patch.object(cli, "_run_behavioral", record), \ + mock.patch.object(cli, "_write_report", lambda *a, **k: None), \ + mock.patch.object(cli, "_prepare_graded_run", lambda *a, **k: ["alpha"]): + yield harness + + +def _behavioral_dataset(slug: str) -> dict: + """`tier0_dataset`, with every triggering case asserting a behaviour. + + `_gradable_cases` keeps only cases with something to grade, so a routing + dataset leaves `cmd_staleness` with nothing to run and it returns before + reaching either half. + """ + data = tier0_dataset(slug) + for evaluation in data[EVALUATIONS_KEY]: + if evaluation[TRIGGER_KEY]: + evaluation["expected_behavior"] = ["do the thing"] + return data + + +def _checked( + case_id: str, + passed: bool, + checks: list[dict], + skill: str = "s", + *, + activated: str | None = "s", + attempt: int = 1, +): + """A behavioral outcome carrying graded checks. + + `activated` defaults to the skill having fired, for the reason `_outcome` + gives. + """ + return behavior.BehaviorOutcome( + id=case_id, skill=skill, prompt="p", passed=passed, elapsed_s=0.0, + checks=checks, activated=activated, attempt=attempt, + ) + + +def _check(text: str, passed: bool, detail: str = ""): + return {"kind": "expected_behavior", "expectation": text, "passed": passed, "detail": detail} + + +class TestBothSidesReasonsSurviveIntoTheReport(unittest.TestCase): + """The first real run of this command failed both ways for *different* + reasons -- the agent tried the task and was blocked on one side, and never + attempted it on the other -- and neither the verdict nor the JSON could + show it. Only the raw inspect logs could, which is not a report.""" + + def _pair(self): + return staleness.pair( + [_checked("c", False, [ + _check("Install it", False, "tried, blocked by missing sudo"), + _check("Avoid the cloud", True), + ])], + [_checked("c", False, [ + _check("Install it", False, "only surveyed, never attempted"), + _check("Avoid the cloud", True), + ])], + ) + + def test_both_sides_checks_reach_the_comparison(self) -> None: + comparison = self._pair()[0] + self.assertEqual(len(comparison.checks_with), 2) + self.assertEqual(len(comparison.checks_without), 2) + + def test_an_expectation_both_sides_met_is_not_reported_as_a_reason(self) -> None: + # Only what went unmet needs explaining. + rows = staleness._reasons(asdict(self._pair()[0])) + self.assertEqual([text for text, _, _ in rows], ["Install it"]) + + def test_the_two_reasons_appear_side_by_side(self) -> None: + summary = staleness.summarize(self._pair(), {"model": "m", "effort": "e"}) + report = staleness.render_markdown(summary) + self.assertIn("blocked by missing sudo", report) + self.assertIn("only surveyed, never attempted", report) + + def test_a_clean_run_is_not_described_as_every_case_needing_the_skill(self) -> None: + # A `no_signal` case has no unmet expectation either, so the empty + # state is about unmet expectations rather than about the verdict. + # The first run with a no_signal case reported the opposite. + comparisons = staleness.pair( + [_checked("c", True, [_check("Do the thing", True)])], + [_checked("c", True, [_check("Do the thing", True)])], + ) + summary = staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + report = staleness.render_markdown(summary) + self.assertEqual(summary["totals"]["no_signal"], 1) + self.assertNotIn("every case needed the skill", report) + self.assertIn("every claim came out the same", report) + + def test_a_check_only_one_side_graded_is_still_shown(self) -> None: + # Fewer checks on one side must not hide the difference. + comparisons = staleness.pair( + [_checked("c", False, [_check("Install it", False, "blocked")])], + [_checked("c", False, [])], + ) + rows = staleness._reasons(asdict(comparisons[0])) + self.assertEqual(rows[0][2], "not graded this side") + + +class _StubSample: + def __init__(self, sample_id, checks, error=None, epoch=1): + self.id = sample_id + self.scores = {"expectations": types.SimpleNamespace(metadata={"checks": checks})} + self.error = error + self.total_time = 1.0 + # inspect stamps the repeat number on every sample. `--attempts` + # above one makes several samples share a case id, and this is what + # tells them apart. + self.epoch = epoch + + +class _StubLog: + def __init__(self, skill, baseline, samples, run_key="run-1", created="2026-01-01"): + metadata = { + engine_behavioral.SKILL_KEY: skill, + engine_behavioral.BASELINE_KEY: baseline, + } + if run_key is not None: + metadata[engine_behavioral.STALENESS_RUN_KEY] = run_key + self.eval = types.SimpleNamespace( + task=f"behavioral-{skill}", metadata=metadata, created=created + ) + self.samples = samples + + +class TestGuardRailsAreNotJudgedAsIfTheyDiscriminated(unittest.TestCase): + """A guard rail holds with or without the skill -- that is its job. + + Folding that into the verdict marks the checks most worth keeping as dead + weight, which would push an author to delete exactly the wrong ones. + """ + + def _guard(self, passed: bool): + return {"kind": "unexpected_behavior", "expectation": "no sudo", + "passed": passed, "detail": ""} + + def test_a_guard_broken_on_an_attempt_that_was_not_decisive_still_counts(self) -> None: + # Guards were read off `checks_with`, which is *one* attempt's + # evidence: `_decisive` picks it on positive claims and firing and + # knows nothing about guards. So above one attempt a skill that ran + # the forbidden thing on an attempt that was not chosen was reported + # as having broken nothing, under a heading calling that the serious + # finding this command produces. + summary = staleness.summarize(staleness.pair( + [_checked("c", True, [_check("do it", True), self._guard(True)], + attempt=1), + _checked("c", False, [_check("do it", False), self._guard(False)], + attempt=2)], + [_checked("c", False, [_check("do it", False), self._guard(True)])], + ), {}) + self.assertEqual(summary["totals"]["guards_broken_with_skill"], 1) + self.assertEqual( + summary["totals"]["guard_checks"], 1, + "one guard rail, counted once however many attempts ran", + ) + self.assertIn("forbidden thing", staleness.render_markdown(summary)) + + def test_a_guard_holding_on_both_sides_does_not_make_a_case_no_signal(self) -> None: + # The claim moved; the guard did not. The verdict follows the claim. + with_skill = _checked("c", True, [_check("run the CLI", True), self._guard(True)]) + without = _checked("c", False, [_check("run the CLI", False), self._guard(True)]) + self.assertEqual(staleness.classify([with_skill], [without]), "earns_it") + + def test_a_case_with_only_guards_is_called_that(self) -> None: + # Nothing was claimed, so nothing can be credited -- but the case is + # not broken, which `fails_either_way` would imply. + both = _checked("c", True, [self._guard(True)]) + self.assertEqual(staleness.classify([both], [both]), "guards_only") + + def test_guards_only_cases_stay_out_of_the_discriminating_share(self) -> None: + comparisons = staleness.pair( + [_checked("a", True, [_check("x", True)]), _checked("b", True, [self._guard(True)])], + [_checked("a", False, [_check("x", False)]), _checked("b", True, [self._guard(True)])], + ) + totals = staleness.summarize(comparisons, {"model": "m", "effort": "e"})["totals"] + self.assertEqual(totals["guards_only"], 1) + self.assertEqual(totals["graded"], 1) + self.assertEqual(totals["discriminating"], 1.0) + + def test_a_guard_broken_with_the_skill_is_surfaced(self) -> None: + # The serious finding: the skill made the agent do the forbidden thing. + comparisons = staleness.pair( + [_checked("c", False, [_check("x", True), self._guard(False)])], + [_checked("c", True, [_check("x", True), self._guard(True)])], + ) + summary = staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + self.assertEqual(summary["totals"]["guards_broken_with_skill"], 1) + self.assertIn("forbidden thing", staleness.render_markdown(summary)) + + def test_the_evidence_shows_even_when_the_skill_earned_it(self) -> None: + # The old report skipped `earns_it` cases, so a run whose whole point + # was that three cases moved printed "nothing to show". + comparisons = staleness.pair( + [_checked("c", True, [_check("ran `rocm --version`", True)])], + [_checked("c", False, [_check("ran `rocm --version`", False)])], + ) + report = staleness.render_markdown( + staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + ) + self.assertIn("What the skill changed", report) + self.assertIn("rocm --version", report) + + +class TestADirtyBaselineCannotShowWhatAnAgentDoesUnaided(unittest.TestCase): + """The first real runs had 40 user-level skills in the baseline room. + + One case invoked an unrelated personal skill and another read the skill + under test straight off the repo; both were reported `no_signal`, which + claims the agent managed unaided. It had not. + """ + + def _pair(self, reason): + without = _checked("c", True, [_check("x", True)]) + without.contaminated = reason + return staleness.pair([_checked("c", True, [_check("x", True)])], [without]) + + def test_a_contaminated_baseline_is_ungraded_not_no_signal(self) -> None: + self.assertEqual(self._pair("another skill fired: oss-contribution")[0].verdict, "ungraded") + + def test_an_earns_it_survives_its_own_contamination(self) -> None: + # The baseline failed while holding more than it should have. It + # fails all the harder without it, so voiding the verdict would throw + # away a conclusion the run is entitled to -- which cost a real run a + # real verdict before this was split out. + without = _checked("c", False, [_check("x", False)]) + without.contaminated = "read the skill's own files (rocm-doctor)" + comparisons = staleness.pair([_checked("c", True, [_check("x", True)])], [without]) + self.assertEqual(comparisons[0].verdict, "earns_it") + + def test_the_report_separates_the_voided_from_the_standing(self) -> None: + clean_fail = _checked("a", False, [_check("x", False)]) + clean_fail.contaminated = "read the skill's own files (s)" + passed_both = _checked("b", True, [_check("y", True)]) + passed_both.contaminated = "another skill fired: helper" + comparisons = staleness.pair( + [_checked("a", True, [_check("x", True)]), _checked("b", True, [_check("y", True)])], + [clean_fail, passed_both], + ) + report = staleness.render_markdown( + staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + ) + self.assertIn("1 of them are reported `ungraded`", report) + self.assertIn("1 still stand", report) + self.assertIn("`a` (earns_it)", report) + + def test_a_clean_baseline_still_reports_no_signal(self) -> None: + self.assertEqual(self._pair(None)[0].verdict, "no_signal") + + def test_the_reason_reaches_the_report(self) -> None: + summary = staleness.summarize( + self._pair("read the skill's own files (quark-torch-llm-ptq)"), + {"model": "m", "effort": "e"}, + ) + report = staleness.render_markdown(summary) + self.assertEqual(summary["totals"]["baseline_contaminated"], 1) + self.assertIn("baseline that was not empty", report) + self.assertIn("read the skill's own files", report) + + def test_the_command_refuses_an_unisolatable_host_run(self) -> None: + # Routing has always refused this; staleness needs it more, because + # the baseline being empty is the measurement rather than an input. + # + # Run, not read. The version of this that grepped `cmd_staleness` for + # "can_isolate_config" passed just as happily with the `not` dropped + # from the guard, which is the one edit that turns the refusal into a + # run that reports a contaminated baseline as a clean one. + with _staleness_harness(self) as harness: + with mock.patch.object(cli.routing, "can_isolate_config", lambda: False): + with self.assertRaises(SystemExit) as caught: + harness.run(engine="claude-code-no-sandbox") + self.assertIn("claude-code-no-sandbox", str(caught.exception)) + self.assertEqual(harness.calls, []) + + def test_an_isolatable_host_run_is_allowed_through(self) -> None: + # The other half of the guard. Without this the test above passes on a + # command that refuses everything. + with _staleness_harness(self) as harness: + with mock.patch.object(cli.routing, "can_isolate_config", lambda: True): + harness.run(engine="claude-code-no-sandbox") + self.assertEqual(len(harness.calls), 2) + + +class TestASkillThatChangedNothingIsToldPlainly(unittest.TestCase): + """`earns_it | 0` is a true row that says nothing, and this is the one + result the command exists to surface.""" + + def _all_no_signal(self): + both = [_checked("a", True, [_check("x", True)])] + return staleness.pair(both, both) + + def test_the_result_is_stated_rather_than_left_in_the_table(self) -> None: + report = staleness.render_markdown( + staleness.summarize(self._all_no_signal(), {"model": "m", "effort": "e"}) + ) + self.assertIn("No case in this suite shows the skill doing anything", report) + + def test_a_case_that_moved_is_not_reported_as_one_that_did_not(self) -> None: + # Found by running the command. A run whose only case was + # `unattributed` printed "every graded case came out the same with + # the skill and without it", which was false of the only case in it: + # it passed with the skill and failed without it. What it lacked was + # evidence the skill was the reason, and `unattributed` says that. + moved = staleness.pair( + [_outcome("a", True, activated=None)], [_outcome("a", False)] + ) + report = staleness.render_markdown( + staleness.summarize(moved, {"model": "m", "effort": "e"}) + ) + self.assertEqual(moved[0].verdict, "unattributed") + self.assertNotIn("No case in this suite shows the skill doing anything", report) + + def test_a_harmful_case_is_not_reported_as_nothing_moving_either(self) -> None: + report = staleness.render_markdown( + staleness.summarize( + staleness.pair([_outcome("a", False)], [_outcome("a", True)]), + {"model": "m", "effort": "e"}, + ) + ) + self.assertNotIn("No case in this suite shows the skill doing anything", report) + + def test_it_names_both_explanations(self) -> None: + # Retire the skill and fix the cases are different actions, and the + # report cannot tell which applies. + report = staleness.render_markdown( + staleness.summarize(self._all_no_signal(), {"model": "m", "effort": "e"}) + ) + self.assertIn("stopped earning its place", report) + self.assertIn("never tested what it knows", report) + + def test_a_run_with_a_discriminating_case_says_nothing_of_the_sort(self) -> None: + comparisons = staleness.pair( + [_checked("a", True, [_check("x", True)])], + [_checked("a", False, [_check("x", False)])], + ) + report = staleness.render_markdown( + staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + ) + self.assertNotIn("shows the skill doing anything", report) + + def test_a_run_that_graded_nothing_does_not_claim_the_skill_is_idle(self) -> None: + # Nothing graded is not the same as nothing changed. + comparisons = staleness.pair([_checked("a", True, [_check("x", True)])], []) + report = staleness.render_markdown( + staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + ) + self.assertNotIn("shows the skill doing anything", report) + + +class TestAFlakyNoSignalIsNotToldToDeleteTheCase(unittest.TestCase): + """`no_signal` tells an author the case is not evidence about the skill. + + That is right when both sides managed it every time and wrong when a + baseline managed it once in three: one baseline pass is enough to deny a + case `earns_it` and nowhere near enough to call it dead. The verdict + cannot tell the two apart, so the report does. + """ + + def _report(self, with_results, without_results): + comparisons = staleness.pair( + [_outcome("c", ok, attempt=i) for i, ok in enumerate(with_results, 1)], + [_outcome("c", ok, attempt=i) for i, ok in enumerate(without_results, 1)], + ) + return staleness.render_markdown(staleness.summarize(comparisons, {})) + + def test_a_side_that_disagreed_with_itself_is_called_out(self) -> None: + report = self._report([True, True, True], [True, False, False]) + self.assertIn("disagreed with itself", report) + self.assertIn("did not settle", report) + + def test_both_sides_managing_it_every_time_keeps_the_old_advice(self) -> None: + report = self._report([True, True], [True, True]) + self.assertNotIn("disagreed with itself", report) + self.assertIn("already did this unaided", report) + + def test_one_attempt_a_side_can_never_read_as_unsettled(self) -> None: + self.assertNotIn("disagreed with itself", self._report([True], [True])) + + +class TestHowMuchOfACaseEachSideMetIsCarried(unittest.TestCase): + """The verdict is binary and the evidence behind it is not. + + A side passes only by meeting every positive claim, so `fails_either_way` + covers a baseline that met four claims of five and one that met none, and + only one of those says the case is worth keeping. Nothing asserted this at + any level: replacing `_claims_met`'s body with `return 0.0` left the whole + suite green. + """ + + def _case(self, with_attempts, without_attempts): + return staleness.pair( + [_checked("c", False, checks, attempt=i) + for i, checks in enumerate(with_attempts, 1)], + [_checked("c", False, checks, attempt=i) + for i, checks in enumerate(without_attempts, 1)], + )[0] + + def test_the_claim_count_belongs_to_the_case_not_to_a_side(self) -> None: + case = self._case( + [[_check("a", True), _check("b", False), _check("c", False)]], + [[_check("a", False), _check("b", False), _check("c", False)]], + ) + self.assertEqual(case.claims, 3) + self.assertEqual((case.claims_met_with, case.claims_met_without), (1.0, 0.0)) + + def test_several_attempts_are_averaged(self) -> None: + # What the pass counts cannot show: a side that met everything once + # and nearly everything the next time. + case = self._case( + [[_check("a", True), _check("b", True)], + [_check("a", True), _check("b", False)]], + [[_check("a", False), _check("b", False)]], + ) + self.assertEqual(case.claims_met_with, 1.5) + + def test_a_graded_side_that_errored_still_lets_the_case_name_its_claims(self) -> None: + # `naming = here or there` could never fall back, because a dataclass + # is always truthy -- so a graded side that errored before grading + # anything printed the case as having claimed nothing, over a baseline + # that had really graded three. + errored = _checked("c", False, [], attempt=1) + errored.error = "API Error: 504" + case = staleness.pair( + [errored], + [_checked("c", False, [ + _check("a", True), _check("b", True), _check("c", False), + ])], + )[0] + self.assertEqual(case.claims, 3) + self.assertEqual(case.claims_met_without, 2.0) + + def test_the_counts_reach_the_report(self) -> None: + case = self._case( + [[_check("a", True), _check("b", False)]], + [[_check("a", False), _check("b", False)]], + ) + report = staleness.render_markdown(staleness.summarize([case], {})) + self.assertIn("1/2 with, 0/2 without", report) + + +class TestAReportWithNothingGradedAssertsNothingAboutGradedCases(unittest.TestCase): + """Sentences whose condition is not the thing they assert. + + The same fault as the note that told a reader nothing had moved on a run + where something had, which a live run caught and a unit test had not. Each + of these printed a fact about graded cases on a run that graded none. + """ + + def _report(self, comparisons): + return staleness.render_markdown(staleness.summarize(comparisons, {})) + + def _guard(self, passed): + return {"kind": "unexpected_behavior", "expectation": "no sudo", + "passed": passed, "detail": "" if passed else "ran sudo"} + + def test_a_guards_only_run_is_not_told_its_graded_cases_came_out_well(self) -> None: + report = self._report(staleness.pair( + [_checked("c", True, [self._guard(True)])], + [_checked("c", True, [self._guard(True)])], + )) + self.assertNotIn("Every graded case either", report) + self.assertIn("No case reached a verdict", report) + + def test_a_broken_guard_is_not_followed_by_nothing_to_look_at(self) -> None: + # The strongest finding this command produces, contradicted two + # paragraphs later by the weakest sentence in it. A `guards_only` case + # is not graded and is not nothing: it ran on both sides and its + # guards were checked. + report = self._report(staleness.pair( + [_checked("c", False, [self._guard(False)])], + [_checked("c", True, [self._guard(True)])], + )) + self.assertIn("the skill doing the forbidden thing", report) + self.assertNotIn("nothing to look at here", report) + self.assertIn("guard rails above are what this run found", report) + + def test_a_run_with_no_guards_either_still_says_there_is_nothing(self) -> None: + report = self._report(staleness.pair([_outcome("c", True)], [])) + self.assertIn("nothing to look at here", report) + + def test_a_case_that_ran_on_one_side_is_not_told_every_case_ran_twice(self) -> None: + report = self._report(staleness.pair([_outcome("c", True)], [])) + self.assertIn("ran on one side only", report) + self.assertNotIn("Every case ran", report) + + def test_nothing_compared_is_not_reported_as_every_claim_matching(self) -> None: + # Both sides stopped before grading anything, so there is no claim to + # have come out the same. "Every claim matched" is a statement about + # the skill; this run never reached one. + sides = [] + for _ in range(2): + attempt = _checked("c", False, [], attempt=1) + attempt.error = "API Error: 504" + sides.append([attempt]) + report = self._report(staleness.pair(*sides)) + self.assertIn("Nothing was compared", report) + self.assertNotIn("every claim came out the same", report) + + def test_a_run_that_did_grade_still_says_so(self) -> None: + report = self._report(staleness.pair([_outcome("c", True)], [_outcome("c", True)])) + self.assertIn("Every case ran twice", report) + self.assertNotIn("nothing to look at here", report) + + +class TestTheCostControlsBindBothHalvesEqually(unittest.TestCase): + """`--max-cases` and `--attempts` decide what a run costs. + + Both halves have to grade the same cases the same number of times. A cap + that gave the baseline a different set, or a repeat count that reached one + leg and not the other, would compare two runs that answered different + questions -- and nothing in the report would say so. + """ + + def _args(self, **kwargs): + defaults = dict( + extended=True, only="", max_cases=0, attempts=1, concurrency=0, + engine="claude-code-no-sandbox", model="opus", effort="high", + ) + return types.SimpleNamespace(**{**defaults, **kwargs}) + + def _cases(self): + return [ + datasets.Case(id=f"{skill}-{i}", skill=skill, prompt="p", + skill_should_trigger=True, expected_behavior=["do it"]) + for skill in ("alpha", "beta") for i in range(3) + ] + + def _select(self, args): + cases = self._cases() + with mock.patch.object( + datasets, "load_dataset", + lambda skill, extended=True: [c for c in cases if c.skill == skill], + ): + return [c.id for c in cli._gradable_cases(args, ["alpha", "beta"])] + + def test_no_cap_grades_everything(self) -> None: + self.assertEqual(len(self._select(self._args())), 6) + + def test_the_cap_is_per_skill(self) -> None: + # Over the whole run instead, a four-skill run would spend its budget + # on the first dataset and report the rest as having nothing to grade. + self.assertEqual( + self._select(self._args(max_cases=2)), + ["alpha-0", "alpha-1", "beta-0", "beta-1"], + ) + + def test_the_cap_is_applied_after_only(self) -> None: + self.assertEqual( + self._select(self._args(max_cases=1, only="alpha-1,alpha-2,beta-0")), + ["alpha-1", "beta-0"], + ) + + def test_both_halves_get_the_same_set(self) -> None: + args = self._args(max_cases=2) + self.assertEqual(self._select(args), self._select(args)) + + def test_the_repeat_count_reaches_the_engine(self) -> None: + # Without this the flag parses, the report says the cases ran several + # times, and inspect ran each of them once. + seen = {} + with mock.patch.object(engine_behavioral, "sandbox_spec"), \ + mock.patch.object(engine_behavioral, "stop_requested", lambda: False), \ + mock.patch.object(engine_behavioral, "build_task", lambda *a, **k: "task"), \ + mock.patch.object(engine_behavioral, "models", mock.MagicMock()), \ + mock.patch.object(engine_behavioral, "run_eval", + lambda ev, task, **kw: seen.update(kw) or []), \ + contextlib.redirect_stdout(io.StringIO()): + engine_behavioral.run( + ["alpha"], self._cases()[:1], "mockllm/model", "high", + solver_factory=None, attempts=3, concurrency=8, + ) + self.assertEqual(seen.get("epochs"), 3) + self.assertEqual(seen.get("max_samples"), 8) + + def test_an_ordinary_run_asks_for_nothing_extra(self) -> None: + # `behavioral` and `routing` must be untouched by flags only + # `staleness` carries. + seen = {} + with mock.patch.object(engine_behavioral, "sandbox_spec"), \ + mock.patch.object(engine_behavioral, "stop_requested", lambda: False), \ + mock.patch.object(engine_behavioral, "build_task", lambda *a, **k: "task"), \ + mock.patch.object(engine_behavioral, "models", mock.MagicMock()), \ + mock.patch.object(engine_behavioral, "run_eval", + lambda ev, task, **kw: seen.update(kw) or []), \ + contextlib.redirect_stdout(io.StringIO()): + engine_behavioral.run( + ["alpha"], self._cases()[:1], "mockllm/model", "high", + solver_factory=None, + ) + self.assertEqual(seen.get("epochs"), 1) + self.assertNotIn("max_samples", seen) + + +class TestAFlagThatCannotDoAnythingIsRefused(unittest.TestCase): + """The repo's habit: refuse an argument rather than accept and ignore it. + + A `--from-logs` rebuild starts no agent. The attempts, the cap and the + concurrency are properties of the run that wrote the logs, so accepting + them would let somebody believe they had re-measured at three attempts + when they had reread one. + """ + + def _run(self, *argv): + parser = cli.build_parser() + with self.assertRaises(SystemExit) as caught: + cli.cmd_staleness(parser.parse_args(["staleness", *argv])) + return str(caught.exception) + + def test_attempts_cannot_change_a_rebuild(self) -> None: + self.assertIn("--attempts", self._run("--from-logs", "d", "--attempts", "3")) + + def test_the_cap_cannot_change_a_rebuild(self) -> None: + self.assertIn("--max-cases", self._run("--from-logs", "d", "--max-cases", "2")) + + def test_a_run_of_no_attempts_is_refused(self) -> None: + self.assertIn("would run nothing", self._run("--attempts", "0")) + + def test_a_negative_count_is_refused(self) -> None: + self.assertIn("cannot be negative", self._run("--concurrency", "-1")) + + def test_choosing_the_evals_cannot_change_a_rebuild(self) -> None: + # Which evals were graded is a property of the run that wrote the + # logs, like the attempts and the cap beside it. It used to be + # accepted and ignored. + self.assertIn( + "--no-extended", self._run("--from-logs", "d", "--no-extended") + ) + + def test_a_harness_with_no_staleness_leg_is_refused(self) -> None: + # The baseline half is only a measurement if the room is empty, and + # the check that establishes that reads Claude Code's credentials and + # no other harness's. Against another agent it would report an + # isolated room it never isolated, which is the one error this whole + # command exists to avoid making. + self.assertIn("no staleness leg", self._run("--harness", "copilot")) + + def test_the_environment_can_name_that_harness_too(self) -> None: + with mock.patch.dict(os.environ, {"SKILLSCOPE_HARNESS": "copilot"}): + self.assertIn("no staleness leg", self._run()) + + def test_the_default_harness_is_not_refused(self) -> None: + # The refusal is about the room check, not about the flag. Naming the + # default explicitly has to behave exactly like not naming it. + said = self._run("--harness", "claude-code", "--from-logs", "/nope") + self.assertNotIn("no staleness leg", said) + + def test_naming_the_agent_cannot_change_a_rebuild(self) -> None: + # A rebuild runs no agent, so which one it would have run is a + # property of the logs. Refused rather than ignored, like the rest. + said = self._run("--from-logs", "d", "--harness", "copilot") + self.assertIn("--harness", said) + self.assertIn("cannot change a --from-logs rebuild", said) + + def test_an_ambient_harness_does_not_refuse_a_rebuild(self) -> None: + # The variable is the shell's, not this invocation's argument. + # Refusing a free rebuild over it would be a refusal nobody asked for. + with mock.patch.dict(os.environ, {"SKILLSCOPE_HARNESS": "copilot"}): + said = self._run("--from-logs", "/nope") + self.assertNotIn("cannot change a --from-logs rebuild", said) + + def test_a_run_id_means_nothing_without_a_rebuild(self) -> None: + # A live run mints its own id and stamps it on both halves, so there + # is nothing here to pick between. + self.assertIn("--from-logs", self._run("--run-id", "abc")) + + +class TestASideCanBeAttemptedMoreThanOnce(unittest.TestCase): + """`--attempts` is what turns a verdict from an anecdote into evidence. + + One run against one run is a coin flip against a coin flip: an agent + disagrees with itself between runs, so a case can pass with the skill and + fail without it for no reason the skill is responsible for. The verdict + rule is unchanged -- a side passed if any attempt met every claim -- so + what the repeats add is the counts beside it. + """ + + def _pair(self, with_results, without_results): + return staleness.pair( + [_outcome("c", ok, attempt=i) for i, ok in enumerate(with_results, 1)], + [_outcome("c", ok, attempt=i) for i, ok in enumerate(without_results, 1)], + )[0] + + def test_the_counts_reach_the_comparison(self) -> None: + case = self._pair([True, True, True], [False, False, False]) + self.assertEqual((case.passes_with, case.attempts_with), (3, 3)) + self.assertEqual((case.passes_without, case.attempts_without), (0, 3)) + + def test_one_attempt_each_is_the_rule_that_was_there_before(self) -> None: + # The property that makes this safe to add: at one attempt a side, + # every verdict is the one the single-pair rule gave. + self.assertEqual(self._pair([True], [False]).verdict, "earns_it") + self.assertEqual(self._pair([True], [True]).verdict, "no_signal") + self.assertEqual(self._pair([False], [True]).verdict, "harmful") + self.assertEqual(self._pair([False], [False]).verdict, "fails_either_way") + + def test_a_side_passes_when_any_attempt_did(self) -> None: + # "Any" rather than "all". A skill that works two times in three has + # done something, and `fails_either_way` would say it had not. + case = self._pair([False, True, False], [False, False, False]) + self.assertEqual(case.verdict, "earns_it") + self.assertEqual(case.passes_with, 1) + + def test_two_flaky_sides_are_no_signal_and_say_so_in_the_counts(self) -> None: + # The verdict cannot tell this from a clean `no_signal`, which is why + # the counts are printed beside it. + case = self._pair([True, False], [False, True]) + self.assertEqual(case.verdict, "no_signal") + self.assertEqual((case.passes_with, case.passes_without), (1, 1)) + + def test_a_baseline_that_managed_it_once_takes_the_case_to_no_signal(self) -> None: + # The rule written down, because nothing asserted it and so the docs + # and the code were free to disagree about it -- `docs/usage.md` and + # `_passes` both said `2/3` against `1/3` was `earns_it`. It is not. + # + # What follows from "any attempt passed" is that a higher attempt + # count is a *higher bar*: `earns_it` at three needs the baseline to + # have failed all three times. So a verdict compares with one taken + # at the same count and not with one taken at another. + self.assertEqual( + self._pair([True, True, False], [False, True, False]).verdict, "no_signal" + ) + self.assertEqual( + self._pair([True, True, True], [False, False, False]).verdict, "earns_it" + ) + self.assertEqual( + self._pair([True, True, True], [True, False, False]).verdict, "no_signal" + ) + + def test_an_errored_attempt_leaves_the_others_their_say(self) -> None: + # One dropped connection of three is not a side that measured + # nothing. The denominator shrinks; the verdict stands on the rest. + case = staleness.pair( + [_outcome("c", True, attempt=1), + _outcome("c", False, attempt=2, error="API Error: 504"), + _outcome("c", True, attempt=3)], + [_outcome("c", False, attempt=i) for i in (1, 2, 3)], + )[0] + self.assertEqual(case.verdict, "earns_it") + self.assertEqual((case.passes_with, case.attempts_with), (2, 2)) + self.assertEqual(case.errored_with, 1) + + def test_a_side_whose_every_attempt_errored_is_ungraded(self) -> None: + case = staleness.pair( + [_outcome("c", False, attempt=i, error="boom") for i in (1, 2)], + [_outcome("c", False, attempt=i) for i in (1, 2)], + )[0] + self.assertEqual(case.verdict, "ungraded") + self.assertEqual((case.attempts_with, case.errored_with), (0, 2)) + + +class TestContaminationVoidsOnTheAttemptTheVerdictRestsOn(unittest.TestCase): + """A dirty baseline voids what it actually refutes, and no more. + + `no_signal` and `harmful` claim the baseline *managed* something unaided. + An attempt it failed cannot be the attempt they rest on, so contamination + there refutes nothing. With one attempt a side the two readings agree; + they part company once a side can be dirty on one attempt and pass on + another. + """ + + def _baseline(self, passed, contaminated, attempt=1): + outcome = _outcome("c", passed, attempt=attempt) + outcome.contaminated = contaminated + return outcome + + def test_a_dirty_baseline_that_passed_voids_no_signal(self) -> None: + case = staleness.pair( + [_outcome("c", True)], [self._baseline(True, "another skill fired: beta")] + )[0] + self.assertEqual(case.verdict, "ungraded") + self.assertTrue(case.baseline_dirty_on_pass) + + def test_a_dirty_baseline_that_failed_leaves_earns_it_standing(self) -> None: + # It fails all the harder without what it should not have had, so + # voiding it would throw away a conclusion the run is entitled to. + case = staleness.pair( + [_outcome("c", True)], [self._baseline(False, "another skill fired: beta")] + )[0] + self.assertEqual(case.verdict, "earns_it") + self.assertFalse(case.baseline_dirty_on_pass) + self.assertTrue(case.baseline_contaminated, "the finding is still reported") + + def test_a_clean_passing_attempt_is_not_voided_by_a_dirty_failed_one(self) -> None: + case = staleness.pair( + [_outcome("c", True, attempt=1)], + [self._baseline(False, "another skill fired: beta", attempt=1), + self._baseline(True, None, attempt=2)], + )[0] + self.assertEqual(case.verdict, "no_signal") + self.assertFalse(case.baseline_dirty_on_pass) + + def test_a_dirty_baseline_that_passed_voids_harmful_too(self) -> None: + # The other half of `VOIDED_BY_CONTAMINATION`, and the one the module + # calls rare and worth stopping for. Nothing asserted it, so dropping + # `harmful` from that tuple left the whole suite green. + case = staleness.pair( + [_outcome("c", False)], + [self._baseline(True, "read the skill's own files (x)")], + )[0] + self.assertEqual(case.verdict, "ungraded") + + def test_a_verdict_resting_on_a_clean_attempt_is_not_called_a_failure(self) -> None: + # The paragraph that explains which verdicts survive contamination + # told the reader they rest on the baseline having *failed*. For this + # case the baseline passed: it was dirty on an attempt it failed and + # clean on the one the verdict rests on. + comparisons = staleness.pair( + [_outcome("c", True, attempt=1)], + [self._baseline(False, "another skill fired: beta", attempt=1), + self._baseline(True, None, attempt=2)], + ) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("rest on a clean attempt", report) + self.assertNotIn("rest on the baseline *failing*", report) + + +class TestEarningItMeansTheSkillActuallyFired(unittest.TestCase): + """Installing a skill does not make an agent use it. + + Until activation was recorded, an `earns_it` meant only that the outcome + differed on the side where the skill happened to be installed. + """ + + def _pair(self, activated): + with_skill = _checked("c", True, [_check("ran the CLI", True)]) + with_skill.activated = activated + return staleness.pair([with_skill], [_checked("c", False, [_check("ran the CLI", False)])]) + + def test_the_firing_is_carried_onto_the_comparison(self) -> None: + self.assertEqual(self._pair("rocm-doctor")[0].activated_with, "rocm-doctor") + + def test_the_order_of_two_passing_attempts_does_not_decide_it(self) -> None: + # The verdict used to be read off `_decisive`'s pick, which is the + # *first* passing attempt -- so the same two runs in the other order + # came out `unattributed`. Which of two attempts happened to run + # first is not a property of the skill. + for order in ((None, "rocm-doctor"), ("rocm-doctor", None)): + with self.subTest(order=order): + case = staleness.pair( + [_checked("c", True, [_check("ran the CLI", True)], + attempt=i, activated=fired) + for i, fired in enumerate(order, 1)], + [_checked("c", False, [_check("ran the CLI", False)])], + )[0] + self.assertEqual(case.verdict, "earns_it") + self.assertEqual(case.activated_with, "rocm-doctor") + + def test_an_attempt_that_fired_and_failed_does_not_credit_the_skill(self) -> None: + # The `_met(a) and` half of the guard, which nothing reached: every + # other test here varies `activated` across attempts that all passed + # or all failed, so the guard never met a counter-example. Removing + # it left the whole suite green. + # + # The skill fired on an attempt the agent *failed*, and the attempt + # that passed never opened it. "The skill fired" and "the skill moved + # the outcome" are the two claims `unattributed` exists to keep apart, + # and this is the shape where they come apart hardest. + case = staleness.pair( + [_checked("c", False, [_check("x", False)], attempt=1, activated="s"), + _checked("c", True, [_check("x", True)], attempt=2, activated=None)], + [_checked("c", False, [_check("x", False)], attempt=1, activated=None)], + )[0] + self.assertEqual(case.verdict, "unattributed") + + def test_no_passing_attempt_firing_is_still_unattributed(self) -> None: + case = staleness.pair( + [_checked("c", True, [_check("ran the CLI", True)], + attempt=i, activated=None) + for i in (1, 2)], + [_checked("c", False, [_check("ran the CLI", False)])], + )[0] + self.assertEqual(case.verdict, "unattributed") + + def test_a_case_that_moved_without_firing_is_not_credited(self) -> None: + # The case passed with the skill installed and failed without it, and + # the agent never opened the skill. Something moved; the thing under + # test is not shown to be what moved it. + totals = staleness.summarize(self._pair(None), {"model": "m", "effort": "e"})["totals"] + self.assertEqual(totals["unattributed"], 1) + self.assertEqual(totals["earns_it"], 0) + + def test_a_case_that_moved_after_firing_is_credited(self) -> None: + totals = staleness.summarize( + self._pair("rocm-doctor"), {"model": "m", "effort": "e"} + )["totals"] + self.assertEqual(totals["earns_it"], 1) + self.assertEqual(totals["unattributed"], 0) + + def test_an_unattributed_case_stays_out_of_the_discriminating_share(self) -> None: + # The share is what the command is read for. Counting a case the skill + # was never shown to have touched would be the one number nobody + # checks saying the thing nobody checked. + totals = staleness.summarize(self._pair(None), {"model": "m", "effort": "e"})["totals"] + self.assertEqual(totals["discriminating"], 0.0) + + def test_the_reader_is_warned_before_believing_the_number(self) -> None: + summary = staleness.summarize(self._pair(None), {"model": "m", "effort": "e"}) + report = staleness.render_markdown(summary) + self.assertIn("the skill never fired on the attempt that passed", report) + self.assertLess( + report.index("never fired on the attempt"), + report.index("### Cases worth a look"), + ) + + def test_a_clean_run_says_nothing_about_firing(self) -> None: + # No warning when every earns_it fired: an unconditional note would be + # noise, and noise is what gets skipped on the run that matters. + report = staleness.render_markdown( + staleness.summarize(self._pair("rocm-doctor"), {"model": "m", "effort": "e"}) + ) + self.assertNotIn("never fired on the attempt", report) + + +class TestAComparisonCanBeRebuiltFromLogs(unittest.TestCase): + """The pairing is written into the logs, so a directory is enough. + + Without this the two runs are related only in the memory of the process + that launched them, and a finished run cannot be looked at again without + paying for it twice over. + """ + + def test_a_log_reports_which_side_it_was(self) -> None: + log = _StubLog("alpha", True, [_StubSample("c", [_check("do it", False)])]) + skill, baseline, _, outcomes = staleness.outcomes_from_log(log) + self.assertEqual(skill, "alpha") + self.assertTrue(baseline) + self.assertEqual(len(outcomes), 1) + self.assertFalse(outcomes[0].passed) + + def test_a_sample_passes_only_when_every_check_did(self) -> None: + log = _StubLog("alpha", False, [ + _StubSample("c", [_check("a", True), _check("b", True)]), + _StubSample("d", [_check("a", True), _check("b", False)]), + ]) + _, _, _, outcomes = staleness.outcomes_from_log(log) + self.assertEqual([o.passed for o in outcomes], [True, False]) + + def test_a_sample_with_no_checks_has_not_passed(self) -> None: + # An ungraded sample is not a green one. + log = _StubLog("alpha", False, [_StubSample("c", [])]) + _, _, _, outcomes = staleness.outcomes_from_log(log) + self.assertFalse(outcomes[0].passed) + + def test_a_rebuild_is_no_more_permissive_than_the_run_it_rebuilds(self) -> None: + # A sample whose checks could not be read is ungraded on the live path + # (`behavioral._outcomes` sets an error for it). Rebuilt without that, + # it comes back as `checks=[]`, which makes `_met()` False, which reads + # as the baseline having failed -- so a harness failure is promoted to + # `earns_it` and inflates the headline number. + # + # Asserted on the verdict, not on the source. The version of this that + # grepped `outcomes_from_log` for "_inspect_room" and "contaminated" + # passed for the whole life of the defect it was named after. + graded = _StubLog("alpha", False, [_StubSample("c", [_check("do it", True)])]) + unreadable = _StubLog("alpha", True, [_StubSample("c", [])]) + _, _, _, with_skill = staleness.outcomes_from_log(graded) + _, _, _, without = staleness.outcomes_from_log(unreadable) + + self.assertEqual( + without[0].error, "case has no behavioral assertions to grade" + ) + self.assertEqual(staleness.classify([with_skill[0]], [without[0]]), "ungraded") + + def test_the_live_path_and_the_rebuild_word_it_the_same_way(self) -> None: + # Two strings that have to match for the two paths to agree, and + # nothing else makes them. + live = inspect.getsource(engine_behavioral._outcomes) + rebuilt = inspect.getsource(staleness.outcomes_from_log) + message = "case has no behavioral assertions to grade" + self.assertIn(message, live) + self.assertIn(message, rebuilt) + + def test_the_two_sides_pair_on_the_skill_they_name(self) -> None: + graded = _StubLog("alpha", False, [_StubSample("c", [_check("do it", True)])]) + baseline = _StubLog("alpha", True, [_StubSample("c", [_check("do it", False)])]) + sides = {} + for log in (graded, baseline): + skill, is_baseline, _, outcomes = staleness.outcomes_from_log(log) + sides.setdefault(skill, {}).setdefault(is_baseline, []).extend(outcomes) + comparisons = staleness.pair(sides["alpha"][False], sides["alpha"][True]) + self.assertEqual(len(comparisons), 1) + # Both sides found each other: one attempt graded on each, and the + # case moved. `unattributed` rather than `earns_it` because a stub log + # carries no tool calls, so nothing in it shows the skill firing -- + # which is the honest reading of a rebuild with no activation + # evidence, and the one `classify` gives. + self.assertEqual(comparisons[0].attempts_with, 1) + self.assertEqual(comparisons[0].attempts_without, 1) + self.assertEqual(comparisons[0].passes_with, 1) + self.assertEqual(comparisons[0].passes_without, 0) + self.assertEqual(comparisons[0].verdict, "unattributed") + + +class TestTheBaselineRunIsTheSameRunWithoutTheSkill(unittest.TestCase): + """The flag has to reach the driver, or the two halves are not comparable.""" + + def _staged(self, *, install: bool) -> list: + """Whether the solver reached `install_skill`, for one value of the flag. + + Driven rather than read. The earlier version of this asserted that the + signature had an `install` parameter defaulting to True, which says + nothing about whether the flag reaches the staging step -- and the + staging step is the entire difference between the two halves. + + The CLI call after it is cut off by giving it no output to parse, which + is the first thing the solver refuses: staging has already happened or + not by then. + """ + import asyncio + + staged: list[Path] = [] + workspace = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, workspace, ignore_errors=True) + + async def no_workspace(): + return workspace + + async def no_output(*args, **kwargs): + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + + solve = engine_no_sandbox.claude_code_no_sandbox( + None, None, Path("alpha"), install=install + ) + state = types.SimpleNamespace(input_text="p", messages=[], store=None) + with mock.patch.object(engine_no_sandbox, "_workspace", no_workspace), \ + mock.patch.object( + engine_no_sandbox, "install_skill", + lambda d, w, skills_dir=None: staged.append(d), + ), \ + mock.patch("inspect_ai.util.subprocess", no_output): + with self.assertRaises(RuntimeError): + asyncio.run(solve(state, None)) + return staged + + def test_the_host_driver_stages_nothing_when_asked_not_to(self) -> None: + self.assertEqual(self._staged(install=False), []) + + def test_the_host_driver_stages_the_skill_when_asked_to(self) -> None: + # The other half: a driver that staged nothing ever would pass the + # test above while making both halves the baseline. + self.assertEqual(self._staged(install=True), [Path("alpha")]) + + def test_the_task_name_says_which_half_it_is(self) -> None: + # Both halves write into one log directory; a name that did not say + # would leave the two indistinguishable on disk. Asserted on the names + # the two calls actually produce, rather than on the source containing + # the suffix somewhere. + repo = Repo(self) + repo.skill("alpha", dataset=tier0_dataset("alpha")) + repo.activate() + cases = [c for c in datasets.load_all_cases() if c.skill == "alpha"][:1] + + graded = engine_behavioral.build_task( + "alpha", cases, "mockllm/model", solver_factory=lambda d: None + ) + baseline = engine_behavioral.build_task( + "alpha", cases, "mockllm/model", + solver_factory=lambda d: None, baseline=True, + ) + self.assertFalse(graded.name.endswith("-baseline")) + self.assertTrue(baseline.name.endswith("-baseline")) + self.assertFalse(graded.metadata[engine_behavioral.BASELINE_KEY]) + self.assertTrue(baseline.metadata[engine_behavioral.BASELINE_KEY]) + + +class TestTheRoomIsJudgedWhereTheRunWasNotWhereTheReaderIs(unittest.TestCase): + """A rebuilt contamination verdict must not depend on who is reading. + + `_skill_marks` recognises a tool call that reached the skill by comparing + against the skill's absolute path. Worked out at read time it is the + *reader's* answer: rebuilt from another directory, or on a machine without + the skill checked out, the mark went missing and a baseline the run had + voided as `ungraded` came back `no_signal` -- stated in the report as + fact. So the run records where the skill was and the reader uses that. + + This is the same hazard `test_a_rebuild_is_no_more_permissive_than_the_run_ + it_rebuilds` guards for the no-checks branch, in the one place it was + still open. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill("run", dataset=tier0_dataset("run")) + self.repo.activate() + self.cases = [c for c in datasets.load_all_cases() if c.skill == "run"][:1] + + def _why(self, arguments, skill_dir=None): + from skillscope.engine import routing as routing_engine + + call = _FakeCall("toolu_1", function="Read") + call.arguments = arguments + sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) + return engine_behavioral._inspect_room( + routing_engine, sample, "run", skill_dir + )[1] + + def test_the_run_records_where_the_skill_was(self) -> None: + # Without this the reader has only its own cwd to go on, and the + # rebuild silently returns to deriving the weaker answer -- which on + # the machine that wrote the logs looks exactly right. + own = str(config.active().skill_path("run").resolve()) + for baseline in (False, True): + task = engine_behavioral.build_task( + "run", self.cases, "mockllm/model", + solver_factory=lambda d: None, baseline=baseline, + ) + self.assertIn( + engine_behavioral.SKILL_DIR_KEY, task.metadata, + "the log does not say where the skill was", + ) + self.assertEqual(task.metadata[engine_behavioral.SKILL_DIR_KEY], own) + + def test_the_other_engine_records_it_too(self) -> None: + # `verify.build_task` needs `inspect_swe`, which the unit job does not + # install, so this is read rather than called -- the same compromise + # `test_the_behavioral_leg_completes_its_skill_too` makes for it. A + # log is paired without knowing which engine wrote it, so a key only + # one of them writes is a key the reader cannot rely on. + self.assertIn( + "SKILL_DIR_KEY", inspect.getsource(engine_verify.build_task) + ) + + def test_the_verdict_is_the_same_from_any_directory(self) -> None: + # The property, asserted as a property. A relative path resolves + # against whoever is running, so resolving one here made the answer a + # fact about the reader's cwd. Standing in the repo root used to say + # contaminated and standing anywhere else said clean, for the same + # recorded call. + own = str(config.active().skill_path("run").resolve()) + argument = {"file_path": "run/SKILL.md"} + + here = os.getcwd() + self.addCleanup(os.chdir, here) + + os.chdir(self.repo.root) + from_the_repo_root = self._why(argument, skill_dir=own) + os.chdir(here) + from_anywhere_else = self._why(argument, skill_dir=own) + + self.assertEqual( + from_the_repo_root, from_anywhere_else, + "the contamination verdict moved with the reader's cwd", + ) + + +class TestTheDetectorKeepsItsTwoAnswersApart(unittest.TestCase): + """`_inspect_room` answers two questions, and they must not cross. + + `fired` is `None` or exactly the skill under test -- `detect_activation` is + called with `skills=[skill]`, so there is nothing else it can legitimately + be. When the two answers shared one `if/elif`, a second `other:` hit + arriving after `why` was already set fell through to the `elif` and was + recorded as the skill having fired. + + That string then reads as truthy everywhere downstream: `activated_with` + on the `Comparison`, and the `earns_it` / `unattributed` split, which + turns on exactly that field. So a case was credited to a skill that never + fired on exactly the contaminated runs that most need the opposite, and + the marker landed verbatim in both JSON reports. + """ + + def _sample(self, calls): + return _FakeSample(messages=[_FakeMessage("m", calls)], events=[]) + + def _call(self, name, arguments=None): + call = _FakeCall("toolu_1", function=name) + call.arguments = arguments or {} + return call + + def _inspect(self, sample, skill="alpha"): + from skillscope.engine import routing as routing_engine + + # A directory for the skill, so the room is knowable and `why` reports + # what this class is about rather than `UNKNOWN_ROOM`. No case here + # names a path, so nothing can match it. + return engine_behavioral._inspect_room( + routing_engine, sample, skill, f"/nonexistent/skills/{skill}" + ) + + def test_a_contamination_marker_is_never_reported_as_the_skill(self) -> None: + # Two `other:` hits. The first sets `why`; the second used to find it + # already set, fail the first condition, and land in `fired`. + sample = self._sample([ + self._call("Skill", {"command": "beta"}), + self._call("Skill", {"command": "gamma"}), + ]) + fired, why = self._inspect(sample) + self.assertIsNone(fired) + self.assertIsNotNone(why) + self.assertIn("another skill fired", why) + + def test_the_skill_under_test_is_still_reported_when_it_fires(self) -> None: + # The other half: a fix that reports nothing would pass the test above. + sample = self._sample([self._call("Skill", {"command": "alpha"})]) + fired, why = self._inspect(sample) + self.assertEqual(fired, "alpha") + self.assertIsNone(why) + + def test_a_marker_cannot_reach_the_report_as_an_activation(self) -> None: + # The consequence, asserted where it is read: a truthy garbage string + # in `activated_with` suppresses the warning this totals. + sample = self._sample([ + self._call("Skill", {"command": "beta"}), + self._call("Skill", {"command": "gamma"}), + ]) + fired, _ = self._inspect(sample) + graded = behavior.BehaviorOutcome( + id="c", skill="alpha", prompt="p", passed=True, elapsed_s=0.0, + checks=[_check("do it", True)], activated=fired, + ) + without = _checked("c", False, [_check("do it", False)], skill="alpha") + totals = staleness.summarize( + staleness.pair([graded], [without]), {"model": "m", "effort": "e"} + )["totals"] + self.assertEqual(totals["unattributed"], 1) + self.assertEqual(totals["earns_it"], 0) + + +class TestContaminationIsJudgedOnTheSkillsRealPath(unittest.TestCase): + """A bare skill name is not evidence that the skill's files were read. + + `structure.py` allows a single-word name and sets no minimum length, and + single-word names are the norm -- Anthropic ship `pdf`, `docx`, `pptx`. The + heuristic matched `/{skill}/` as a substring of every tool call's + serialised arguments, so a skill named `run` was reported as contaminated + by any agent that touched `/run/user/1000`, and `docs`, `test`, `tools` or + `scripts` by any ordinary path. + + The cost of that false positive is the highest there is here: + `_voided_if_dirty` converts `no_signal` and `harmful` -- the two verdicts + the command exists to surface -- into `ungraded`, and the report states the + contamination as fact. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill("run", dataset=tier0_dataset("run")) + self.repo.activate() + + def _why(self, arguments, skill="run", skill_dir=None): + from skillscope.engine import routing as routing_engine + + call = _FakeCall("toolu_1", function="Read") + call.arguments = arguments + sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) + return engine_behavioral._inspect_room( + routing_engine, sample, skill, skill_dir + )[1] + + def test_an_unrelated_path_sharing_the_skills_name_is_not_contamination(self) -> None: + self.assertIsNone(self._why({"file_path": "/run/user/1000/bus"})) + + def test_a_bash_command_mentioning_the_name_is_not_contamination(self) -> None: + self.assertIsNone( + self._why({"command": "ls /run/systemd && mkdir -p build/run/out"}) + ) + + def test_the_skills_own_directory_still_counts(self) -> None: + own = str(config.active().skill_path("run") / "SKILL.md") + self.assertIn("read the skill's own files", self._why({"file_path": own})) + + def test_the_staged_copy_still_counts(self) -> None: + staged = "/tmp/ws-1234/.claude/skills/run/reference.md" + self.assertIn("read the skill's own files", self._why({"file_path": staged})) + + def test_a_bash_command_naming_the_real_directory_still_counts(self) -> None: + own = str(config.active().skill_path("run")) + self.assertIn( + "read the skill's own files", self._why({"command": f"cat {own}/SKILL.md"}) + ) + + def test_the_harness_staging_directory_still_counts(self) -> None: + self.assertIn( + "read the skill's own files", + self._why({"file_path": "/opt/_bundled_skills/run/SKILL.md"}), + ) + + def test_a_sibling_skill_whose_name_extends_this_one_is_not_this_one(self) -> None: + # The false positive that survived the first fix in the shell branch, + # because the staged mark carried no trailing boundary there while + # the structured branch used `_under`. `run-extra` is its own skill. + self.assertIsNone( + self._why({"command": "cat /home/u/.claude/skills/run-extra/SKILL.md"}) + ) + self.assertIsNone( + self._why({"file_path": "/home/u/.claude/skills/run-extra/SKILL.md"}) + ) + + def test_a_name_the_skills_own_name_is_a_prefix_of_is_not_a_hit(self) -> None: + own = str(config.active().skill_path("run")) + self.assertIsNone(self._why({"file_path": f"{own}.bak/SKILL.md"})) + + def test_the_staged_copy_is_found_by_the_relative_name(self) -> None: + # `no_sandbox` runs the agent with `cwd=workspace`, so this is the + # natural shell form. A mark anchored with a leading separator missed + # it, and a missed contamination is the worse direction: the baseline + # is then reported clean, as fact. + self.assertIn( + "read the skill's own files", + self._why({"command": "cat .claude/skills/run/SKILL.md"}), + ) + + def test_an_argument_the_field_list_never_named_is_still_read(self) -> None: + # `pattern` is Glob's primary argument and is often its only one, so + # the old fixed field list made the call invisible. The tool surface + # is open, so the list was always going to drift behind it. + own = str(config.active().skill_path("run")) + self.assertIn( + "read the skill's own files", self._why({"pattern": f"{own}/**/*.md"}) + ) + + def test_a_tool_naming_its_arguments_anything_is_still_read(self) -> None: + # An MCP tool names its arguments whatever it likes. + own = str(config.active().skill_path("run")) + self.assertIn( + "read the skill's own files", self._why({"some_mcp_arg": f"{own}/x.md"}) + ) + + def test_a_path_nested_inside_a_structured_argument_is_still_read(self) -> None: + own = str(config.active().skill_path("run")) + self.assertIn( + "read the skill's own files", + self._why({"edits": [{"file": f"{own}/SKILL.md"}]}), + ) + + def test_a_skill_with_no_directory_does_not_take_the_command_down(self) -> None: + # `config.skill_path` raises `SystemExit`, which `except Exception` + # does not catch -- so rebuilding somebody else's logs killed the + # process rather than reporting what it could. + why = self._why({"file_path": "/tmp/x"}, skill="not-a-skill") + self.assertIsNotNone(why) + + def test_a_skill_with_no_directory_is_an_unknown_room_not_a_clean_one(self) -> None: + # It used to answer `None` here, which reads as "the baseline was + # clean" -- a verdict reached by not looking. The mark it dropped is + # the only one an absolute path can hit, so every tool call that + # reached the skill by its real path came back clean on any machine + # without the skill checked out. + why = self._why({"file_path": "/tmp/x"}, skill="not-a-skill") + self.assertEqual(why, engine_behavioral.UNKNOWN_ROOM) + + def test_a_recorded_directory_answers_it_without_the_skill_on_this_machine( + self, + ) -> None: + # The point of recording it: the same log, read where the skill is not. + own = str(config.active().skill_path("run")) + self.assertIn( + "read the skill's own files", + self._why( + {"file_path": f"{own}/SKILL.md"}, skill="not-a-skill", skill_dir=own + ), + ) + + +class TestAWalkThatDidNotFinishCannotVouchForTheRoom(unittest.TestCase): + """An interrupted walk inspected only the calls it got to. + + `_inspect_room` computes `unknown` once, before the walk, from whether + there were marks to check against. That answers "could the read half of + the check run at all", not "did the check actually run", and the `except` + branch used to return the same expression a completed walk returns. So a + walk that died on its second call reported the room clean, as fact, with + the contamination sitting on the third call unread. + + `_activation_event` reaches `call.function` with no `getattr` guard, + unlike every other accessor on this path, so an unexpected call shape is + the realistic way in -- and `--from-logs`, reading logs this code did not + write, is exactly where an unexpected shape would come from. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill("run", dataset=tier0_dataset("run")) + self.repo.activate() + + def test_a_walk_cut_short_is_an_unknown_room_not_a_clean_one(self) -> None: + from skillscope.engine import routing as routing_engine + + class _Unwalkable: + id = "toolu_2" + arguments: dict = {} + + @property + def function(self): + raise AttributeError("a shape this code did not write") + + innocuous = _FakeCall("toolu_1", function="Read") + innocuous.arguments = {"file_path": "/tmp/notes.md"} + # Never reached, and the whole point: the evidence the walk would + # have found had it got there. + unread = _FakeCall("toolu_3", function="Read") + own = config.active().skill_path("run") / "SKILL.md" + unread.arguments = {"file_path": str(own)} + + sample = _FakeSample( + messages=[_FakeMessage("m", [innocuous, _Unwalkable(), unread])], + events=[], + ) + _fired, why = engine_behavioral._inspect_room( + routing_engine, sample, "run", None + ) + self.assertEqual(why, engine_behavioral.UNKNOWN_ROOM) + + +class TestBothHalvesRunAndOnlyTheInstallFlagDiffers(unittest.TestCase): + """The command's whole measurement is that the two runs differ in one way. + + Called rather than read. Every earlier assertion about `cmd_staleness` was + a substring check against its source, and `cli._run_behavioral` was never + invoked by a test at all. + """ + + def test_the_cases_are_run_twice(self) -> None: + with _staleness_harness(self) as harness: + self.assertEqual(harness.run(), 0) + self.assertEqual(len(harness.calls), 2) + + def test_the_two_halves_disagree_about_installing(self) -> None: + # The regression this exists for: two calls with the same `install` + # compare a run against itself and report `no_signal` for everything. + with _staleness_harness(self) as harness: + harness.run() + self.assertEqual([c["install"] for c in harness.calls], [True, False]) + + def test_both_halves_grade_the_same_cases(self) -> None: + with _staleness_harness(self) as harness: + harness.run() + self.assertEqual(harness.calls[0]["cases"], harness.calls[1]["cases"]) + self.assertTrue(harness.calls[0]["cases"]) + + def test_both_halves_carry_one_run_key(self) -> None: + # What `--from-logs` pairs on. Two different keys leave the halves + # unpairable; no key leaves them pairable with anything. + with _staleness_harness(self) as harness: + harness.run() + keys = {c["run_key"] for c in harness.calls} + self.assertEqual(len(keys), 1) + self.assertTrue(keys.pop()) + + def test_two_invocations_do_not_share_a_run_key(self) -> None: + with _staleness_harness(self) as harness: + harness.run() + harness.run() + self.assertNotEqual(harness.calls[0]["run_key"], harness.calls[2]["run_key"]) + + def test_a_dataset_with_nothing_to_grade_runs_neither_half(self) -> None: + with _staleness_harness(self, dataset=tier0_dataset("alpha")) as harness: + self.assertEqual(harness.run(), 0) + self.assertEqual(harness.calls, []) + + +class TestARebuildPairsOneRunAndNotItsNeighbours(unittest.TestCase): + """`.skillscope/logs` is shared, never pruned, and gitignored wholesale. + + Every ordinary `behavioral` run stamps `BASELINE_KEY: False`, so grouping + on `(skill, baseline)` alone made an unrelated run of the same skill a + candidate for this comparison's "with skill" half -- rendered into a table + that reads exactly as confidently as a real one. `docs/usage.md` documents + pointing `--from-logs` at that very directory. + """ + + def _rebuild(self, logs, unlabelled=True, run_id=None): + return self._read(logs, unlabelled, run_id).comparisons + + def _read(self, logs, unlabelled=True, run_id=None): + info = [types.SimpleNamespace(name=f"log-{i}") for i in range(len(logs))] + by_name = {i.name: log for i, log in zip(info, logs)} + with mock.patch("inspect_ai.log.list_eval_logs", lambda d: info), \ + mock.patch("inspect_ai.log.read_eval_log", lambda n: by_name[n]): + return staleness.from_logs( + "anywhere", unlabelled=unlabelled, run_id=run_id + ) + + def _log(self, baseline, passed, run_key="run-1", case="c", created="2026-01-01"): + return _StubLog( + "alpha", baseline, + [_StubSample(case, [_check("do it", passed)])], + run_key=run_key, + created=created, + ) + + def test_one_runs_two_halves_pair(self) -> None: + comparisons = self._rebuild([self._log(False, True), self._log(True, False)]) + # One case, graded on both sides, that passed only with the skill. A + # stub log has no tool calls to show the skill firing, so the verdict + # is `unattributed`; what this is about is that the two halves found + # each other at all. + self.assertEqual( + [(c.passes_with, c.passes_without, c.verdict) for c in comparisons], + [(1, 0, "unattributed")], + ) + + def test_a_neighbouring_runs_log_is_not_spliced_in(self) -> None: + # The stale graded half says the case passed; this run's says it + # failed. Pooled, the pair reads `earns_it` off a run nobody asked + # about. The newest run is read, and only its own halves reach it. + rebuild = self._read([ + self._log(False, True, run_key="old", created="2026-01-01"), + self._log(False, False, run_key="new", created="2026-02-02"), + self._log(True, False, run_key="new", created="2026-02-02"), + ]) + self.assertEqual( + [(c.run, c.verdict) for c in rebuild.comparisons], + [("new", "fails_either_way")], + ) + self.assertEqual(rebuild.runs, {"alpha": ["new", "old"]}) + + def _two_runs(self): + return [ + self._log(False, True, run_key="first", created="2026-01-01"), + self._log(True, False, run_key="first", created="2026-01-01"), + self._log(False, True, run_key="second", created="2026-03-03"), + self._log(True, False, run_key="second", created="2026-03-03"), + ] + + def test_two_runs_of_one_skill_are_not_both_reported(self) -> None: + # The run id stopped two runs being *spliced*; it did not stop them + # being *pooled*. A one-case skill came back as a two-case suite with + # the same row twice, on the second staleness run anybody did, under + # the invocation `docs/usage.md` gives. + rebuild = self._read(self._two_runs()) + self.assertEqual([c.run for c in rebuild.comparisons], ["second"]) + self.assertEqual(rebuild.runs, {"alpha": ["second", "first"]}) + + def _across_skills(self, *specs): + return [ + _StubLog(skill, baseline, + [_StubSample(f"c-{skill}", [_check("do it", passed)])], + run_key=run, created=created) + for skill, baseline, passed, run, created in specs + ] + + def test_a_run_read_for_one_skill_is_not_called_unread_by_another(self) -> None: + # One invocation mints one run id and stamps it on every skill it + # covers, so a run id spans skills. Asked per skill, `RUN1` was read + # for `beta` and announced as unread by `alpha`, two lines under the + # sentence naming it as read. + rebuild = self._read(self._across_skills( + ("alpha", False, True, "RUN1", "2026-01-01"), + ("alpha", True, False, "RUN1", "2026-01-01"), + ("beta", False, True, "RUN1", "2026-01-01"), + ("beta", True, False, "RUN1", "2026-01-01"), + ("alpha", False, True, "RUN2", "2026-03-03"), + ("alpha", True, False, "RUN2", "2026-03-03"), + )) + report = staleness.render_markdown(staleness.summarize( + rebuild.comparisons, {"from_logs": "/l", "runs": rebuild.runs}, + )) + self.assertIn("run `RUN2` for `alpha`", report) + self.assertIn("run `RUN1` for `beta`", report) + self.assertNotIn("were not read", report) + + def test_an_unread_run_is_listed_with_the_skill_it_belongs_to(self) -> None: + # `--run-id` is global, so an id is only actionable beside the skill + # whose run it was. + rebuild = self._read(self._across_skills( + ("alpha", False, True, "A1", "2026-01-01"), + ("alpha", True, False, "A1", "2026-01-01"), + ("alpha", False, True, "A2", "2026-03-03"), + ("alpha", True, False, "A2", "2026-03-03"), + ("beta", False, True, "B1", "2026-02-02"), + ("beta", True, False, "B1", "2026-02-02"), + )) + report = staleness.render_markdown(staleness.summarize( + rebuild.comparisons, {"from_logs": "/l", "runs": rebuild.runs}, + )) + self.assertIn("1 other staleness run(s)", report) + self.assertIn("`A1` for `alpha`", report) + + def test_an_older_run_can_be_asked_for_by_id(self) -> None: + rebuild = self._read(self._two_runs(), run_id="first") + self.assertEqual([c.run for c in rebuild.comparisons], ["first"]) + + def test_an_id_that_names_no_run_reads_nothing_and_says_what_there_is(self) -> None: + rebuild = self._read(self._two_runs(), run_id="third") + self.assertEqual(rebuild.comparisons, []) + self.assertEqual(rebuild.runs, {"alpha": ["second", "first"]}) + + def test_the_report_names_the_run_it_read_and_what_it_passed_over(self) -> None: + rebuild = self._read(self._two_runs()) + report = staleness.render_markdown(staleness.summarize( + rebuild.comparisons, {"from_logs": "/l", "runs": rebuild.runs}, + )) + self.assertIn("run `second` for `alpha`", report) + self.assertIn("1 other staleness run(s)", report) + self.assertIn("`first`", report) + + def test_unlabelled_logs_are_not_read_unless_asked_for(self) -> None: + # The default. A plain `behavioral` log carries `BASELINE_KEY: False` + # and no run id, which is the same shape a pre-id graded half has, so + # reading unlabelled logs unasked pools whatever is in the directory. + comparisons = self._rebuild([ + self._log(False, True, run_key=None), + self._log(True, False, run_key=None), + ], unlabelled=False) + self.assertEqual(comparisons, []) + + def test_an_unlabelled_pair_is_read_when_asked_for(self) -> None: + # What `--unlabelled-logs` is for: one log a side, nothing to choose + # between, and a baseline half that only a staleness run ever writes. + comparisons = self._rebuild([ + self._log(False, True, run_key=None), + self._log(True, False, run_key=None), + ]) + self.assertEqual( + [(c.passes_with, c.passes_without, c.verdict) for c in comparisons], + [(1, 0, "unattributed")], + ) + + def test_repeats_of_one_case_are_not_read_as_a_collision(self) -> None: + # What `--attempts` writes: one log a side, each case answered once + # per attempt. Keyed on the case id alone this looked exactly like two + # logs fighting over one case, and every case came back `ungraded`. + comparisons = self._rebuild([ + _StubLog("alpha", False, [ + _StubSample("c", [_check("do it", True)], epoch=1), + _StubSample("c", [_check("do it", True)], epoch=2), + ]), + _StubLog("alpha", True, [ + _StubSample("c", [_check("do it", False)], epoch=1), + _StubSample("c", [_check("do it", False)], epoch=2), + ]), + ]) + self.assertEqual( + [(c.passes_with, c.attempts_with, c.passes_without, c.attempts_without) + for c in comparisons], + [(2, 2, 0, 2)], + ) + + def test_two_logs_answering_the_same_attempt_are_still_refused(self) -> None: + # The ambiguity the guard exists for, which repeats must not hide: + # which of the two is this side's answer cannot be told, so neither + # is kept. + graded = _StubLog("alpha", False, [_StubSample("c", [_check("do it", True)])]) + comparisons = self._rebuild([ + graded, + _StubLog("alpha", False, [_StubSample("c", [_check("do it", True)])]), + _StubLog("alpha", True, [_StubSample("c", [_check("do it", False)])]), + ]) + self.assertEqual([c.verdict for c in comparisons], ["ungraded"]) + self.assertIn("same attempt", comparisons[0].error) + + def test_graded_only_unlabelled_logs_are_never_read(self) -> None: + # The common hole. `behavioral` and `verify` runs never carry a run + # id by design, so these keep arriving forever -- and pooled they + # inflated `cases` and `ungraded` with every skill the user had ever + # run `behavioral` against, plus a `per_skill` row for it. + # + # Asking for unlabelled logs does not buy this: a baseline half is + # the one thing nothing but `cmd_staleness` writes, so requiring one + # keeps ordinary behavioral logs out whatever flags are passed. + for asked in (True, False): + with self.subTest(unlabelled=asked): + comparisons = self._rebuild( + [self._log(False, True, run_key=None)], unlabelled=asked + ) + self.assertEqual(comparisons, []) + + def test_a_behavioral_log_cannot_supply_a_half_to_a_stale_baseline(self) -> None: + # The rarer hole, and the original splicing bug through the back door: + # a plain `behavioral` log for a skill, plus a leftover pre-id + # baseline for the same skill, is one log a side and was paired. + # `summarize` counts what comes back, so this has to stay empty. + comparisons = self._rebuild([ + self._log(False, True, run_key=None), # indistinguishable from + # a plain behavioral run + self._log(True, False, run_key=None), + ], unlabelled=False) + self.assertEqual(staleness.summarize(comparisons, {})["totals"]["cases"], 0) + + def test_unlabelled_logs_with_a_choice_to_make_are_not_guessed_at(self) -> None: + # Two candidates for the graded side and no way to tell which run + # either belongs to. That ambiguity is the whole hazard, so it is + # reported rather than silently resolved to the older log. + comparisons = self._rebuild([ + self._log(False, True, run_key=None), + self._log(False, False, run_key=None, case="d"), + self._log(True, False, run_key=None), + ]) + self.assertTrue(comparisons) + self.assertEqual({c.verdict for c in comparisons}, {"ungraded"}) + self.assertIn("no run id", comparisons[0].error) + + def test_a_labelled_run_never_falls_back_to_the_unlabelled_bucket(self) -> None: + # A labelled half and an unlabelled one are not two halves of + # anything, however tempting the arithmetic. The labelled run is read + # and reported as the lone half it is, rather than paired into an + # `earns_it`; `LEGACY_RUN` is a bucket of whatever carried no id, so + # it never outranks a run however recently its logs were written. + rebuild = self._read([ + self._log(False, True, run_key="new", created="2026-01-01"), + self._log(True, False, run_key=None, created="2026-09-09"), + ]) + self.assertEqual( + [(c.run, c.verdict) for c in rebuild.comparisons], [("new", "ungraded")] + ) + self.assertIn("no result from", rebuild.comparisons[0].error) + self.assertEqual(rebuild.runs, {"alpha": ["new", staleness.LEGACY_RUN]}) + + def test_two_logs_reporting_one_case_are_reported_rather_than_resolved(self) -> None: + # `pair` builds a dict keyed by case id, so a duplicate collapses + # silently -- and `list_eval_logs` returns newest first, which makes + # the survivor the older of the two. + comparisons = self._rebuild([ + self._log(False, True), + self._log(False, False), + self._log(True, False), + ]) + self.assertEqual([c.verdict for c in comparisons], ["ungraded"]) + self.assertIn("more than one log", comparisons[0].error) + + +class TestAStalenessRunHonoursHooksTheSameWayBehavioralDoes(unittest.TestCase): + """It drives the same `build_task`, so it cannot skip what that refuses to. + + `_require_hook_support` guarded on `command != "behavioral"`, and + `_prepare_graded_run` passes `args.command` -- so under `staleness` the + guard returned before it checked anything. A skill whose `evals/hooks.py` + defines `check` or `setup_session` had them silently skipped, on both + halves, which is the exact "dangerous half" the function exists to + prevent: the case is graded as though the hook ran. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill( + "alpha", + dataset=tier0_dataset("alpha"), + hooks="def check(run):\n return True\n", + ) + self.repo.activate() + + def _refused(self, command): + try: + cli._require_hook_support("claude-code", ["alpha"], command) + except SystemExit as caught: + return str(caught) + return None + + def test_a_staleness_run_is_refused_the_same_as_a_behavioral_one(self) -> None: + refusal = self._refused("staleness") + self.assertIsNotNone(refusal) + self.assertIn("check", refusal) + + def test_the_behavioral_refusal_is_unchanged(self) -> None: + self.assertIsNotNone(self._refused("behavioral")) + + def test_routing_is_still_exempt(self) -> None: + # It installs the skills, asks which one fired, and executes nothing. + self.assertIsNone(self._refused("routing")) + + +class TestAnOfflineRebuildDoesNotClaimAModelItNeverRan(unittest.TestCase): + """`--from-logs` reaches no model, so it has no model to report. + + The meta dict passed `args.model` / `args.effort` straight through, and + nothing was read off the logs -- so rebuilding a run taken on `sonnet` at + `low` with a bare `--from-logs` printed "on `opus` (effort `high`)". Not a + hedge: a false statement about where a measurement came from, in the one + mode that never runs `enforce_model_policy` either. + """ + + def _report(self, meta): + comparisons = staleness.pair( + [_checked("c", True, [_check("do it", True)])], + [_checked("c", False, [_check("do it", False)])], + ) + return staleness.render_markdown(staleness.summarize(comparisons, meta)) + + def test_a_rebuild_names_no_model(self) -> None: + report = self._report({"from_logs": "/logs", "skills": ["alpha"]}) + headline = report.split("\n\n")[1] + self.assertNotIn("(effort", headline) + self.assertNotIn(" on `", headline) + self.assertIn("Rebuilt offline", report) + self.assertIn("/logs", report) + + def test_a_live_run_still_names_its_model(self) -> None: + report = self._report({"model": "sonnet", "effort": "low"}) + self.assertIn("`sonnet`", report) + self.assertIn("effort `low`", report) + + def test_the_command_passes_no_model_through_on_a_rebuild(self) -> None: + captured: dict = {} + real = staleness.summarize + + def record(comparisons, meta): + captured.update(meta) + return real(comparisons, meta) + + args = cli.build_parser().parse_args( + ["staleness", "--from-logs", "/logs", "--model", "opus"] + ) + with mock.patch.object(staleness, "summarize", record), \ + mock.patch.object( + staleness, "from_logs", + lambda d, **k: staleness.Rebuild(staleness.pair( + [_checked("c", True, [_check("x", True)])], + [_checked("c", False, [_check("x", False)])], + )), + ), \ + mock.patch.object(cli, "_write_report", lambda *a, **k: None): + cli.cmd_staleness(args) + self.assertNotIn("model", captured) + self.assertNotIn("effort", captured) + self.assertEqual(captured["from_logs"], "/logs") + + +class TestUnlabelledLogsAreOptedIntoRatherThanAssumed(unittest.TestCase): + """The flag has to reach `from_logs`, and mean nothing without a rebuild.""" + + def _run(self, argv): + captured: dict = {} + + def record(log_dir, *, unlabelled=False, run_id=None): + captured["unlabelled"] = unlabelled + captured["run_id"] = run_id + return staleness.Rebuild( + staleness.pair( + [_checked("c", True, [_check("x", True)])], + [_checked("c", False, [_check("x", False)])], + ) + ) + + args = cli.build_parser().parse_args(argv) + with mock.patch.object(staleness, "from_logs", record), \ + mock.patch.object(cli, "_write_report", lambda *a, **k: None): + cli.cmd_staleness(args) + return captured + + def test_a_rebuild_does_not_read_unlabelled_logs_by_default(self) -> None: + self.assertIs(self._run(["staleness", "--from-logs", "/l"])["unlabelled"], False) + + def test_the_flag_reaches_the_rebuild(self) -> None: + self.assertIs( + self._run( + ["staleness", "--from-logs", "/l", "--unlabelled-logs"] + )["unlabelled"], + True, + ) + + def test_the_flag_is_refused_without_a_rebuild(self) -> None: + # A live run stamps a run id on both halves, so there is nothing + # unlabelled for this to reach. Refused rather than ignored. + args = cli.build_parser().parse_args(["staleness", "--unlabelled-logs"]) + with self.assertRaises(SystemExit) as caught: + cli.cmd_staleness(args) + self.assertIn("--from-logs", str(caught.exception)) + + +class TestARebuildIsNarrowedRatherThanIgnored(unittest.TestCase): + """`--skill` and `--only` were accepted and ignored under `--from-logs`. + + The command refuses `--attempts` with a rebuild on the reasoning that an + argument which cannot do anything should be refused rather than accepted. + These two can do something: a shared log directory holding several skills + is exactly where a reader wants to ask about one of them. + """ + + def _rebuilder(self): + def rebuild(log_dir, **kwargs): + return staleness.Rebuild(staleness.pair( + [_checked("a", True, [_check("x", True)], "alpha"), + _checked("b", True, [_check("x", True)], "beta")], + [_checked("a", False, [_check("x", False)], "alpha"), + _checked("b", False, [_check("x", False)], "beta")], + )) + return rebuild + + def _summary(self, *argv): + captured: dict = {} + args = cli.build_parser().parse_args( + ["staleness", "--from-logs", "/l", *argv] + ) + with mock.patch.object(staleness, "from_logs", self._rebuilder()), \ + mock.patch.object( + cli, "_write_report", + lambda summary, *a, **k: captured.update(summary), + ): + cli.cmd_staleness(args) + return captured + + def test_the_whole_directory_is_reported_when_nothing_narrows_it(self) -> None: + self.assertEqual(self._summary()["meta"]["skills"], ["alpha", "beta"]) + + def test_a_skill_narrows_what_is_reported(self) -> None: + self.assertEqual(self._summary("--skill", "alpha")["meta"]["skills"], ["alpha"]) + + def test_a_case_id_narrows_what_is_reported(self) -> None: + self.assertEqual( + [c["id"] for c in self._summary("--only", "b")["cases"]], ["b"] + ) + + def test_a_filter_matching_nothing_is_refused_rather_than_reported_empty(self) -> None: + args = cli.build_parser().parse_args( + ["staleness", "--from-logs", "/l", "--skill", "gamma"] + ) + with mock.patch.object(staleness, "from_logs", self._rebuilder()), \ + mock.patch.object(cli, "_write_report", lambda *a, **k: None), \ + self.assertRaises(SystemExit) as caught: + cli.cmd_staleness(args) + self.assertIn("gamma", str(caught.exception)) + + def test_a_run_id_that_misses_a_skill_says_which(self) -> None: + # `--run-id` is global and a run id is not: naming one drops every + # skill its invocation did not cover. The "no such run" guard cannot + # catch it, because it only fires when nothing matched at all. + def rebuild(log_dir, **kwargs): + return staleness.Rebuild( + staleness.pair( + [_checked("a", True, [_check("x", True)], "alpha")], + [_checked("a", False, [_check("x", False)], "alpha")], + ), + {"alpha": ["RUN2", "RUN1"], "beta": ["RUN1"]}, + ) + + args = cli.build_parser().parse_args( + ["staleness", "--from-logs", "/l", "--run-id", "RUN2"] + ) + said = io.StringIO() + with mock.patch.object(staleness, "from_logs", rebuild), \ + mock.patch.object(cli, "_write_report", lambda *a, **k: None), \ + contextlib.redirect_stdout(said): + self.assertEqual(cli.cmd_staleness(args), 0) + self.assertIn("beta", said.getvalue()) + self.assertIn("did not cover", said.getvalue()) + + def test_a_run_id_covering_every_skill_says_nothing(self) -> None: + def rebuild(log_dir, **kwargs): + return staleness.Rebuild( + staleness.pair( + [_checked("a", True, [_check("x", True)], "alpha")], + [_checked("a", False, [_check("x", False)], "alpha")], + ), + {"alpha": ["RUN2", "RUN1"]}, + ) + + args = cli.build_parser().parse_args( + ["staleness", "--from-logs", "/l", "--run-id", "RUN2"] + ) + said = io.StringIO() + with mock.patch.object(staleness, "from_logs", rebuild), \ + mock.patch.object(cli, "_write_report", lambda *a, **k: None), \ + contextlib.redirect_stdout(said): + cli.cmd_staleness(args) + self.assertEqual(said.getvalue(), "") + + def test_an_expired_deadline_still_fails_a_rebuild(self) -> None: + # `cmd_staleness`'s docstring says so without qualification, and this + # branch alone returned 0 without ever asking. Reading a large + # directory is not free. + args = cli.build_parser().parse_args(["staleness", "--from-logs", "/l"]) + with mock.patch.object(staleness, "from_logs", self._rebuilder()), \ + mock.patch.object(cli, "_write_report", lambda *a, **k: None), \ + mock.patch.object(cli, "_fail_if_expired", lambda: 3): + self.assertEqual(cli.cmd_staleness(args), 3) + + +class TestTheSecondPassGetsABudgetOfItsOwn(unittest.TestCase): + """Two passes under one bound, and the baseline is the slower half. + + Measured on a two-case skill: the graded pass finished both cases in + 2m23s, and the baseline had not finished the second after 30 minutes. An + agent holding the skill goes straight to the work; an agent without it + explores. On the default 900s the baseline got ~637s of a documented + 1800s+ need, which makes truncation the expected outcome on the command's + own canonical example rather than a risk. + """ + + def _timeout(self, argv): + return cli.build_parser().parse_args(argv).timeout + + def test_staleness_defaults_to_twice_the_shared_bound(self) -> None: + self.assertEqual( + self._timeout(["staleness"]), 2 * deadline.DEFAULT_TIMEOUT_S + ) + + def test_the_single_pass_commands_are_unchanged(self) -> None: + self.assertEqual( + self._timeout(["behavioral"]), deadline.DEFAULT_TIMEOUT_S + ) + self.assertEqual(self._timeout(["routing"]), deadline.DEFAULT_TIMEOUT_S) + + def test_an_explicit_timeout_still_wins(self) -> None: + self.assertEqual(self._timeout(["staleness", "--timeout", "60"]), 60.0) + + def test_the_help_text_names_the_default_actually_in_force(self) -> None: + # The help was a static string built in the shared helper, so the + # `set_defaults` override changed the value and left `--help` saying + # 900 -- three lines under a description announcing that staleness + # defaults to twice the usual. One screen contradicting itself. + import contextlib + import io + + def help_for(command): + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + with self.assertRaises(SystemExit): + cli.build_parser().parse_args([command, "--help"]) + return " ".join(buffer.getvalue().split()) + + self.assertIn("Default: 1800 (30 minutes)", help_for("staleness")) + self.assertIn("Default: 900 (15 minutes)", help_for("behavioral")) + + def test_the_help_text_lists_staleness_as_a_command_sharing_the_bound(self) -> None: + import contextlib + import io + + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + with self.assertRaises(SystemExit): + cli.build_parser().parse_args(["staleness", "--help"]) + self.assertIn("behavioral and staleness", " ".join(buffer.getvalue().split())) + + + if __name__ == "__main__": unittest.main(verbosity=2) From 205f57d22654089989637f2dbc1a7c9f6926b7e8 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Wed, 11 Nov 2026 16:46:40 +0100 Subject: [PATCH 02/17] A judge that could not answer is not a skill that failed `judge.grade` returned `(False, reason)` when the sandbox would not list and when the grader answered in prose. Both are statements about the run, and both arrived as a check that simply failed. In `behavioral` that is a red case. In `staleness` it is the headline. `answered()` keeps any attempt without an `error`, so a baseline whose judge broke counted as a baseline that genuinely failed, which is `earns_it`, and a graded run whose judge broke counted as `harmful`. The two verdicts the command exists to produce were the two a grader hiccup manufactured. The polarity is where it bites hardest: on `must_happen=False` a `False` asserts the forbidden thing happened, so a grader that rambled was reported as the agent having done it. `judge.Ungradable` carries the distinction now. An exception rather than a changed return type, because the caller has to do something different with it and a `bool | None` invites being read as falsy by every caller that does not know yet. Checks gain a `graded` flag, read through `scorers.graded()` so a dict from a log written before this still reads correctly, and one ungraded check errors the attempt: a side passes only by meeting every positive claim, so a claim with no verdict leaves the side undecidable rather than negative. `staleness.answered()` needs no change. It already drops attempts with an error, which is why that was the channel to use. --- skillscope/engine/behavioral.py | 12 ++ skillscope/engine/judge.py | 24 +++- skillscope/engine/scorers.py | 56 ++++++++-- skillscope/staleness.py | 7 ++ tests/test_skillscope.py | 187 ++++++++++++++++++++++++++++++++ 5 files changed, 275 insertions(+), 11 deletions(-) diff --git a/skillscope/engine/behavioral.py b/skillscope/engine/behavioral.py index b271d79..3837a5f 100644 --- a/skillscope/engine/behavioral.py +++ b/skillscope/engine/behavioral.py @@ -420,6 +420,18 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: error = f"{sample.error.message}" elif not checks: error = "case has no behavioral assertions to grade" + elif ungraded := [c for c in checks if not scorers.graded(c)]: + # One check nobody could grade makes the whole attempt + # unanswerable rather than partly answered: a side passes only by + # meeting *every* positive claim, so a claim with no verdict + # leaves the result undecidable, not negative. Reported as an + # error so it lands in the errors column and gets triaged as + # infrastructure, which is what it is -- and so that + # `staleness.answered` drops the attempt instead of reading a + # grader that never answered as evidence about a skill. + error = "could not be graded: " + "; ".join( + c["detail"] for c in ungraded[:3] + ) activated, contaminated = _inspect_room(routing_engine, sample, skill) diff --git a/skillscope/engine/judge.py b/skillscope/engine/judge.py index f4f0a30..9046f7a 100644 --- a/skillscope/engine/judge.py +++ b/skillscope/engine/judge.py @@ -45,6 +45,24 @@ """ +class Ungradable(Exception): + """The judge could not reach a verdict. The run's failure, not the skill's. + + Raised rather than returned as `(False, reason)`. A `False` here is a + statement about what the agent did, and an infrastructure fault wearing + that shape is not a neutral mistake -- it is one that reliably produces + the most alarming reading available. `skillscope staleness` compares two + runs, so a baseline whose judge never answered reads as a case the skill + was needed for, and a graded run whose judge never answered reads as a + skill that did harm. Both are conclusions about a skill drawn from a + sandbox that would not list or a grader that returned prose. + + An exception rather than a third return value because the caller has to + do something different with it, and a `bool | None` invites being read as + falsy by every caller that does not know yet. + """ + + def is_image(path: str) -> bool: return PurePosixPath(path).suffix.lower() in IMAGE_SUFFIXES @@ -232,7 +250,7 @@ async def grade( # Say so rather than presenting an empty workspace as fact. A judge told # "no files" will confidently report the agent did nothing, which reads # as the skill failing when the sandbox is what failed. - return False, f"judge skipped: could not list the sandbox -- {exc}" + raise Ungradable(f"could not list the sandbox -- {exc}") from exc described, images = await artifacts(paths) @@ -277,7 +295,9 @@ async def grade( parsed = parse_verdict(output.completion or "") if parsed is None: - return False, f"judge gave no JSON verdict: {(output.completion or '')[:200]!r}" + raise Ungradable( + f"judge gave no JSON verdict: {(output.completion or '')[:200]!r}" + ) satisfied, reason = parsed return satisfied, f"judge: {reason}" diff --git a/skillscope/engine/scorers.py b/skillscope/engine/scorers.py index 1b3bb09..63e063e 100644 --- a/skillscope/engine/scorers.py +++ b/skillscope/engine/scorers.py @@ -22,15 +22,37 @@ CHECKS = "checks" -def _check(kind: str, expectation: str, passed: bool, detail: str = "") -> dict: +def _check( + kind: str, + expectation: str, + passed: bool, + detail: str = "", + graded: bool = True, +) -> dict: + """One expectation's result. `graded` False means nobody could tell. + + `passed` stays False where nothing graded it, so a reader that knows only + about pass and fail still refuses to call the case green. Anything + attributing a result to the skill has to read `graded` first. + + Always written rather than omitted when True, so the dict has one shape. + Readers still default it to True -- see `graded()` -- because a dict read + back from a log written before this existed does not carry the key. + """ return { "kind": kind, "expectation": expectation, "passed": passed, "detail": detail, + "graded": graded, } +def graded(check: dict) -> bool: + """Whether anybody actually graded this check. A missing key reads as True.""" + return bool(check.get("graded", True)) + + def searchable(state) -> str: """Everything in the run, for `logs_contain` to search. @@ -81,7 +103,13 @@ async def score(state, target) -> "Score": # would blame the agent for the harness's failure. for path in wanted: checks.append( - _check("files_exist", path, False, f"could not list the sandbox: {exc}") + _check( + "files_exist", + path, + False, + f"could not list the sandbox: {exc}", + graded=False, + ) ) files = None else: @@ -98,13 +126,23 @@ async def score(state, target) -> "Score": # Judged expectations last: the deterministic results are on screen # before the grader calls, which take a few seconds each, begin. - for statement in meta.get(convert.EXPECTED, []): - ok, reason = await judge.grade(statement, state, must_happen=True) - checks.append(_check("expected_behavior", statement, ok, reason)) - - for statement in meta.get(convert.UNEXPECTED, []): - ok, reason = await judge.grade(statement, state, must_happen=False) - checks.append(_check("unexpected_behavior", statement, ok, reason)) + for kind, key, must_happen in ( + ("expected_behavior", convert.EXPECTED, True), + ("unexpected_behavior", convert.UNEXPECTED, False), + ): + for statement in meta.get(key, []): + try: + ok, reason = await judge.grade( + statement, state, must_happen=must_happen + ) + except judge.Ungradable as exc: + # The polarity is why this cannot be left as a False. + # On `must_happen=False` a False asserts the forbidden + # thing *happened*, so a grader that never answered + # would be reported as the agent having done it. + checks.append(_check(kind, statement, False, str(exc), graded=False)) + continue + checks.append(_check(kind, statement, ok, reason)) passed = bool(checks) and all(c["passed"] for c in checks) return Score( diff --git a/skillscope/staleness.py b/skillscope/staleness.py index 08b40f2..1c8776b 100644 --- a/skillscope/staleness.py +++ b/skillscope/staleness.py @@ -200,6 +200,13 @@ def answered(attempts: list[BehaviorOutcome]) -> list[BehaviorOutcome]: denominator would report a provider that dropped the connection as a side that failed, which is the most flattering possible reading of an error on the graded side and the least flattering on the baseline. + + A grader that could not reach a verdict sets `error` too, from + `engine/behavioral._outcomes`. That is the same category and the one most + likely to be mistaken for a result: a sandbox that would not list, or a + judge that answered in prose, used to arrive here as a check that simply + failed, which manufactured `earns_it` out of a broken baseline and + `harmful` out of a broken graded run. """ return [a for a in attempts if not a.error] diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 06ca6a9..13b4327 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -5081,6 +5081,193 @@ def test_no_jobs_means_inspect_decides(self) -> None: self.assertIsNone(self.seen.get("max_samples")) +class TestAJudgeThatNeverAnsweredIsNotASkillThatEarnedIt( + unittest.IsolatedAsyncioTestCase +): + """A grader that could not answer used to be reported as the agent failing. + + `judge.grade` returned `(False, "judge skipped: ...")` when the sandbox + would not list, and `(False, "judge gave no JSON verdict: ...")` when the + grader answered in prose. Both are statements about the run, and both + arrived as a check that simply failed. + + In `behavioral` that is merely a red case. In `staleness` it is the + headline: `answered()` keeps any attempt without an `error`, so a baseline + whose judge broke counted as a baseline that genuinely failed, which is + `earns_it`, and a graded run whose judge broke counted as a skill doing + harm. The two verdicts the command exists to produce were the two a + grader hiccup manufactured. + """ + + async def test_a_sandbox_that_will_not_list_is_not_a_verdict(self) -> None: + async def _boom(): + raise engine_tools.ListingFailed("no container") + + with mock.patch.object(engine_judge, "tools_list_paths", _boom): + with self.assertRaises(engine_judge.Ungradable): + await engine_judge.grade("it wrote a file", object(), must_happen=True) + + async def test_a_grader_answering_in_prose_is_not_a_verdict(self) -> None: + # The polarity is why this one matters most. On `must_happen=False` a + # `False` asserts the forbidden thing *happened*, so a grader that + # rambled used to be reported as the agent having done it. + async def _none(): + return [] + + class _Model: + async def generate(self, _messages): + return types.SimpleNamespace(completion="I could not decide.") + + with mock.patch.object(engine_judge, "tools_list_paths", _none), \ + mock.patch("inspect_ai.model.get_model", lambda *a, **k: _Model()): + with self.assertRaises(engine_judge.Ungradable): + await engine_judge.grade( + "it called a cloud API", _JudgeState(), must_happen=False + ) + + +class _JudgeState: + """The little of `TaskState` that `judge.grade` reads to build its evidence.""" + + messages: list = [] + output = None + + +class TestAnUngradedCheckIsAnErrorNotAFailure(unittest.TestCase): + """The attempt is unanswerable, not failed, and the difference is the point. + + A side passes only by meeting *every* positive claim, so one claim with no + verdict leaves the side undecidable rather than negative. Routed through + `error` because that is the channel `staleness.answered` already drops, and + because it puts the fault in the errors column where it gets triaged as + infrastructure instead of read as the skill violating an expectation. + """ + + @staticmethod + def _case(case_id: str): + return datasets.Case( + id=case_id, skill="alpha", prompt="p", skill_should_trigger=True, + ) + + @staticmethod + def _log(checks): + sample = types.SimpleNamespace( + id="c1", + scores={"s": types.SimpleNamespace( + metadata={engine_scorers.CHECKS: checks} + )}, + error=None, + total_time=1.0, + ) + return types.SimpleNamespace( + status="success", error=None, samples=[sample] + ) + + def test_a_check_nobody_graded_errors_the_attempt(self) -> None: + log = self._log([ + {"kind": "expected_behavior", "expectation": "e", "passed": False, + "detail": "could not list the sandbox -- no container", + "graded": False}, + ]) + outcome = engine_behavioral._outcomes(log, "alpha", [self._case("c1")])[0] + self.assertIn("could not be graded", outcome.error or "") + self.assertFalse(outcome.passed) + + def test_an_ordinary_failure_is_still_a_failure(self) -> None: + # The other direction, and the one a careless fix breaks: a case that + # genuinely failed its expectation must stay a failure, not become an + # error that `staleness` then declines to count at all. + log = self._log([ + {"kind": "expected_behavior", "expectation": "e", "passed": False, + "detail": "judge: it did not", "graded": True}, + ]) + outcome = engine_behavioral._outcomes(log, "alpha", [self._case("c1")])[0] + self.assertIsNone(outcome.error) + self.assertFalse(outcome.passed) + + def test_a_check_from_an_older_log_reads_as_graded(self) -> None: + # Logs written before `graded` existed have no such key, and must not + # all become ungraded the moment this lands. + self.assertTrue(engine_scorers.graded({"passed": True, "kind": "k"})) + log = self._log([{"kind": "k", "expectation": "e", "passed": True, + "detail": ""}]) + outcome = engine_behavioral._outcomes(log, "alpha", [self._case("c1")])[0] + self.assertIsNone(outcome.error) + self.assertTrue(outcome.passed) + + +class TestABrokenJudgeOnTheBaselineIsNotEvidenceAboutTheSkill(unittest.TestCase): + """The regression itself, stated as the verdict it used to manufacture. + + This is the one that would have shipped: a baseline attempt whose grader + never answered, paired against a graded attempt that passed, reported as + `earns_it` -- the skill credited for a case the baseline was never + actually asked. + """ + + # Both halves are built by running `_outcomes` over a log, not by handing + # `pair` an outcome with `error` already set. Setting it by hand tests + # that `classify` honours an error it was given, which was never the bug + # and passes with the fix reverted. The claim here is about the chain: an + # ungraded check has to *become* that error before `pair` ever sees it. + # + # The module-level `_outcome` helper is used where a plain outcome is + # wanted, rather than a method of this class: `self._outcome` is where + # `unittest.TestCase` keeps the running test's `_Outcome`, so a helper by + # that name is shadowed during the run while looking right on the class. + + @staticmethod + def _side(check: dict) -> list: + sample = types.SimpleNamespace( + id="c1", + scores={"s": types.SimpleNamespace( + metadata={engine_scorers.CHECKS: [check]} + )}, + error=None, + total_time=1.0, + ) + log = types.SimpleNamespace(status="success", error=None, samples=[sample]) + case = datasets.Case( + id="c1", skill="alpha", prompt="p", skill_should_trigger=True, + ) + return engine_behavioral._outcomes(log, "alpha", [case]) + + def test_a_baseline_nobody_could_grade_comes_out_ungraded(self) -> None: + with_skill = self._side( + {"kind": "expected_behavior", "expectation": "e", "passed": True, + "detail": "judge: it did", "graded": True} + ) + without = self._side( + {"kind": "expected_behavior", "expectation": "e", "passed": False, + "detail": "could not list the sandbox -- no container", + "graded": False} + ) + comparison = staleness.pair(with_skill, without)[0] + self.assertEqual( + comparison.verdict, "ungraded", + "a baseline whose judge never answered is not a baseline that failed", + ) + + def test_a_baseline_that_really_failed_is_still_graded(self) -> None: + # The control. Without it the fix could pass by calling everything + # ungraded, which would make the command useless rather than wrong. + # + # Asserted as "graded, and the skill is the side that passed" rather + # than as a verdict name: which name a case like this carries is what + # the taxonomy collapse is about, and this test is about the judge. + with_skill = self._side( + {"kind": "expected_behavior", "expectation": "e", "passed": True, + "detail": "judge: it did", "graded": True} + ) + without = self._side( + {"kind": "expected_behavior", "expectation": "e", "passed": False, + "detail": "judge: it did not", "graded": True} + ) + comparison = staleness.pair(with_skill, without)[0] + self.assertNotEqual(comparison.verdict, "ungraded") + self.assertEqual((comparison.passes_with, comparison.passes_without), (1, 0)) + + class TestOneBadSampleDoesNotFailTheWholeBatch(unittest.TestCase): """A behavioral batch discarded every case it had already graded. From 90bbfb3ba133b024548bbd0a06b6ff12daba1b70 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Wed, 11 Nov 2026 17:20:43 +0100 Subject: [PATCH 03/17] Drop the offline rebuild, to come back on its own `--from-logs`, `--run-id` and `--unlabelled-logs` rebuilt a comparison from logs already on disk. The feature is sound and will return, but it arrived welded to the command it reads for, and reviewers asked for the smaller thing first: one PR that runs two halves and reports what differed. Most of the complexity it carried was its own. A log directory is shared with `behavioral` and `routing`, is never pruned, and holds every run anybody has done in it, so the rebuild needed a run id to pair on, a rule for picking one run per skill, a flag for logs written before the id existed, and a way to report a side it could not answer for. None of that is needed to run two halves and compare them. `STALENESS_RUN_KEY` goes with it. Nothing read it once `outcomes_from_log` was gone, and a key written for a reader that does not exist is the kind of dead weight this round of review was about. Removing it also narrows the `build_task`/`run` keyword surface that #39 touches. `test_the_default_harness_is_not_refused` is fixed rather than deleted. It passed `--from-logs`, which returned before the harness guard was reached, so it asserted the absence of a refusal that could not have fired and would have gone on passing had the guard been written to refuse the default too. staleness.py 1314 -> 975 lines. --- docs/usage.md | 72 +--- skillscope/cli.py | 244 +----------- skillscope/engine/behavioral.py | 21 -- skillscope/engine/verify.py | 12 +- skillscope/staleness.py | 355 +----------------- tests/test_skillscope.py | 631 +------------------------------- 6 files changed, 26 insertions(+), 1309 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 1de7a86..1b3daef 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -326,68 +326,16 @@ without it explores. Three flags decide what a run costs. | `--max-cases N` | every case | Grade only the first N cases of each selected skill, after `--only` and `--extended` have chosen them. Both sides run the same N, and the report says the run did not cover the whole suite. | | `--concurrency N` | inspect's default | How many cases may be in flight at once. Where the wall clock goes on the baseline half. Raise it as far as the engine's sandboxes and the model's rate limits allow. | -None of the three reaches a `--from-logs` rebuild, which runs no agent, and -passing one with it is refused rather than ignored. `--no-extended` is refused -with it for the same reason: which evals were graded is a property of the run -that wrote the logs. `--skill` and `--only` do work on a rebuild, where they -narrow what is reported rather than what is run. - -Both halves write which side they were, whose skill they were about, where that -skill was on disk, and which invocation wrote them into their inspect log -metadata (`skillscope_baseline`, `skillscope_skill`, `skillscope_skill_dir`, -`skillscope_staleness_run`). So a finished run can be looked at again without -paying for it twice: - -```bash -skillscope staleness --from-logs .skillscope/logs -``` - -That reaches no model and runs no preflight, having nothing to spend. - -A rebuild reads **one run per skill, the most recent**. Nothing prunes -`.skillscope/logs`, so after your second staleness run the directory holds two -of everything, and reading both would report every case once per run and count -it that many times in the totals: a one-case skill would come back as a -two-case suite with the same row printed twice. The report names the run it -read and lists the ids it passed over. `--run-id` reads one of those instead: - -```bash -skillscope staleness --from-logs .skillscope/logs --run-id 7f3a9c21 -``` - -Every key is load-bearing. `skillscope_skill_dir` is what lets a rebuild judge -whether the baseline was really empty: the check recognises a tool call that -reached the skill by comparing against the skill's absolute path, so a reader -that worked that path out for itself would be working out its own rather than -the run's, and would report a baseline the run had voided as a clean one. A log -written without the key is read as an unknown room rather than an empty one. - -The run id is the key that makes this safe to point at `.skillscope/logs`. That directory is shared with `behavioral` -and `routing`, is gitignored wholesale, and nothing prunes it — and every -ordinary `behavioral` run writes the same `skillscope_baseline: false` a graded -half does, with no run id. A plain `behavioral` log is therefore shaped exactly -like a graded half, and always will be, since only `staleness` mints an id. -Pairing on the skill alone would splice an unrelated run from last month in as -this comparison's "with skill" side, under a table that reads exactly as -confidently as a real one. - -So logs carrying no run id are passed over, and a directory of plain -`behavioral` logs rebuilds as nothing rather than as a comparison. Logs written -before the id existed are passed over for the same reason; `--unlabelled-logs` -reads them: - -```bash -skillscope staleness --from-logs .skillscope/logs --unlabelled-logs -``` - -That asserts the directory holds nothing else for the skills in it. Even then, -a group of unlabelled logs needs a baseline half, which nothing but `staleness` -ever writes, so ordinary `behavioral` logs stay out whatever flags you pass. -Where one side holds several unlabelled logs there is a choice to make and -nothing to make it with, so every case in that group is reported `ungraded`. - -The model and effort are not reported for a rebuild: nothing reads them off -the logs, and the logs need not agree on them. +Both halves write which side they were, whose skill they were about, and where +that skill was on disk into their inspect log metadata (`skillscope_baseline`, +`skillscope_skill`, `skillscope_skill_dir`). + +`skillscope_skill_dir` is the load-bearing one. It is what lets the baseline be +judged empty: the check recognises a tool call that reached the skill by +comparing against the skill's absolute path, so a reader working that path out +for itself would be working out its own rather than the run's, and would report +a baseline the run had voided as a clean one. A log written without the key is +read as an unknown room rather than an empty one. Budget more wall clock than a `behavioral` run of the same cases, and more than double it. `--timeout` defaults to twice what the single-pass commands diff --git a/skillscope/cli.py b/skillscope/cli.py index 90817c3..4cad3c4 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -60,9 +60,6 @@ # without it, twice the agent passes of a behavioral run skillscope staleness --skill serving-llms-on-epyc - # the same comparison again, read back off the logs it already wrote - skillscope staleness --from-logs .skillscope/logs - # what CI should run for a change skillscope select --since BASE HEAD @@ -81,7 +78,6 @@ import sys import tempfile import time -import uuid from pathlib import Path from . import ( @@ -849,7 +845,6 @@ def _run_behavioral( *, install: bool = True, config_dir: Path | None = None, - run_key: str | None = None, ) -> list: """Run the behavioral cases on the selected engine. @@ -858,10 +853,6 @@ def _run_behavioral( similar call sites, so the only difference between the run being measured and the run it is measured against is the flag. - `run_key` is stamped on the logs both halves write, so `--from-logs` can - find this run's two halves and only this run's. Set by `cmd_staleness` - alone -- see `engine/behavioral.py:STALENESS_RUN_KEY`. - `attempts` and `concurrency` are read off `args` rather than passed, so the two halves cannot be given different ones. A baseline attempted fewer times than the graded half is not a baseline, and the counts the report @@ -882,7 +873,6 @@ def _run_behavioral( engine_models.resolve(args.model), args.effort, install=install, - run_key=run_key, attempts=attempts, concurrency=concurrency, ) @@ -907,7 +897,6 @@ def _run_behavioral( subject, args.model, args.effort, d, config_dir, install=install ), baseline=not install, - run_key=run_key, attempts=attempts, concurrency=concurrency, ) @@ -988,7 +977,7 @@ def cmd_staleness(args: argparse.Namespace) -> int: started = time.time() asked = getattr(args, "harness", None) or os.environ.get("SKILLSCOPE_HARNESS") - if not args.from_logs and asked and asked != harnesses.DEFAULT: + if asked and asked != harnesses.DEFAULT: # The one place this command cannot be harness-neutral yet. Its # baseline half is only a measurement if the room is empty, and the # check that establishes that -- `routing.can_isolate_config` below -- @@ -1006,26 +995,6 @@ def cmd_staleness(args: argparse.Namespace) -> int: "`routing` and `behavioral`, which are harness-neutral." ) - if args.run_id and not args.from_logs: - # Same habit as `--unlabelled-logs` below. A live run mints its own - # run id and stamps it on both halves, so there is no run here for a - # reader to choose between. - raise SystemExit( - "error: --run-id only means something with --from-logs. A run " - "started now mints its own id, and there is nothing to pick " - "between." - ) - - if args.unlabelled_logs and not args.from_logs: - # The repo's habit: refuse an argument that cannot do anything rather - # than accept and ignore it. A live run writes its own run id on both - # halves, so there is nothing unlabelled for this to reach. - raise SystemExit( - "error: --unlabelled-logs only means something with --from-logs. " - "A run started now stamps a run id on both halves, so it has no " - "unlabelled logs to read." - ) - if args.attempts < 1: raise SystemExit( f"error: --attempts {args.attempts} would run nothing. The " @@ -1038,162 +1007,6 @@ def cmd_staleness(args: argparse.Namespace) -> int: "cannot be negative. 0 means no cap on either." ) - # Same habit again. None of these reaches a rebuild: `--from-logs` starts - # no agent, and the attempts, the cap, the concurrency and which evals - # were graded are all properties of the run that wrote the logs rather - # than of the reader. Accepting them silently would let somebody believe - # they had re-measured something at three attempts when they had reread - # one attempt. - # - # `--skill` and `--only` are not on this list, and used to be ignored like - # it. They name a subset of what is already there, which a reader of a - # shared log directory has every reason to want, so the rebuild filters on - # them below instead. - spent = [ - name - for name, value in ( - ("--attempts", args.attempts > 1), - ("--max-cases", args.max_cases), - ("--concurrency", args.concurrency), - ("--no-extended", not args.extended), - # The flag, not `asked`: an ambient `SKILLSCOPE_HARNESS` is not an - # argument to this invocation, and refusing a free rebuild over a - # variable the shell happens to export would be a refusal nobody - # asked for. - ("--harness", args.harness), - ) - if value - ] - if args.from_logs and spent: - raise SystemExit( - f"error: {', '.join(spent)} cannot change a --from-logs rebuild. " - "It runs no agent: the attempts, the cases, the concurrency, the " - "agent and which evals were graded are whatever the logs were " - "written with. " - "The report reads them back off the logs. Use --skill or --only " - "to narrow what is reported." - ) - - if args.from_logs: - # Nothing is run, so nothing is prepared: no structural gate, no model - # pin, no reachability probe. Those guard a spend that is not about to - # happen, and demanding them would make a free command fail for - # reasons that cannot affect its answer. - rebuild = staleness.from_logs( - args.from_logs, - unlabelled=args.unlabelled_logs, - run_id=args.run_id or None, - ) - comparisons = rebuild.comparisons - known = sorted({run for ids in rebuild.runs.values() for run in ids}) - if args.run_id and not comparisons and known: - # Told apart from "no staleness logs here at all", because they - # are different mistakes: this one is a typo in an id, and the - # answer to it is the list of ids that would have worked. - raise SystemExit( - f"error: no staleness run `{args.run_id}` in " - f"{args.from_logs}. The runs there are: " - + ", ".join(f"`{run}`" for run in known) - + "." - ) - if args.run_id: - # `--run-id` is global and a run id is not. One invocation mints - # one id and stamps it on whichever skills it was asked for, so - # naming an id silently drops every skill that invocation did not - # cover -- and the guard above cannot catch it, because it only - # fires when *nothing* matched. Asking for one run and getting one - # run is legitimate; losing a skill without being told is not. - covered = {c.skill for c in comparisons} - missing = sorted(set(rebuild.runs) - covered) - if missing: - print( - f"[staleness] run {args.run_id} covers " - f"{len(covered)} of the {len(rebuild.runs)} skill(s) in " - f"{args.from_logs}. Nothing is reported for " - + ", ".join(missing) - + ": that run did not cover them. Each has runs of its " - "own; drop --run-id to read the most recent of each.", - flush=True, - ) - - # `--skill` and `--only` narrow what is reported rather than what is - # read. A log directory is shared and accumulates, so a reader asking - # about one skill in it is the ordinary case; ignoring the flags, as - # this used to, answered a question nobody asked. - wanted = {s.strip() for s in args.skill.split(",") if s.strip()} - cases = {c.strip() for c in args.only.split(",") if c.strip()} - if wanted or cases: - comparisons = [ - c - for c in comparisons - if (not wanted or c.skill in wanted) - and (not cases or c.id in cases) - ] - if not comparisons: - raise SystemExit( - "error: nothing in " - f"{args.from_logs} matches " - + " and ".join( - part - for part in ( - f"--skill {args.skill}" if wanted else "", - f"--only {args.only}" if cases else "", - ) - if part - ) - + "." - ) - if not comparisons: - from .engine import behavioral as engine_behavioral - - raise SystemExit( - f"error: no skillscope staleness logs in {args.from_logs}. " - f"Both halves of a comparison carry `{engine_behavioral.SKILL_KEY}`, " - f"`{engine_behavioral.BASELINE_KEY}` and " - f"`{engine_behavioral.STALENESS_RUN_KEY}` in their eval " - "metadata; logs written by another tool, or by a plain " - "`behavioral` run, cannot be paired." - + ( - "" - if args.unlabelled_logs - else "\n Logs written before the run id existed carry " - "none, and are passed over. `--unlabelled-logs` reads " - "them, where a baseline half is there to pair with." - ) - ) - # No `model`, `engine` or `effort`. Nothing here read them off the - # logs, and `args` holds this invocation's defaults -- which describe - # the command that did the rebuilding, not the runs being rebuilt. The - # report prints what it is given, so passing them through made it - # assert a model the measurement was never taken on. - summary = staleness.summarize( - comparisons, - { - "skills": sorted({c.skill for c in comparisons}), - "from_logs": args.from_logs, - # The runs behind what is being reported, so the report can - # name the one it read and say what it passed over. Narrowed - # to the skills that survived the filters above: a skill the - # reader excluded has runs that were not read, and counting - # those as passed over would answer a question nobody asked - # with a number nobody wanted. See `staleness.Rebuild`. - "runs": { - skill: ids - for skill, ids in rebuild.runs.items() - if skill in {c.skill for c in comparisons} - }, - "run_id": args.run_id or None, - "wall_time_s": round(time.time() - started, 1), - "github_run_id": os.environ.get("GITHUB_RUN_ID"), - }, - ) - _write_report(summary, staleness.render_markdown(summary), args, "staleness") - # The docstring says a deadline that expired still fails, and said it - # of this branch too while this branch alone returned without asking. - # Reading a large directory is not free, and a command that overran - # the bound it was given should say so whatever it spent the time on. - return _fail_if_expired() or 0 - skills = _prepare_graded_run(args) gradable = _gradable_cases(args, skills) @@ -1224,12 +1037,6 @@ def cmd_staleness(args: argparse.Namespace) -> int: "guest has no user-level skills at all." ) - # One token for this invocation, stamped on every log both halves write. - # `--from-logs` has nothing else to pair on: the log directory is shared - # and never pruned, and a plain `behavioral` run stamps the same - # `BASELINE_KEY: False` the graded half does, so without this an unrelated - # old run is a candidate for this comparison's "with skill" side. - run_key = uuid.uuid4().hex passes = ( "run twice" if args.attempts <= 1 @@ -1250,11 +1057,11 @@ def cmd_staleness(args: argparse.Namespace) -> int: host = args.engine == "claude-code-no-sandbox" with_skill = _run_behavioral( args, skills, gradable, - config_dir=Path(graded_dir) if host else None, run_key=run_key, + config_dir=Path(graded_dir) if host else None, ) without = _run_behavioral( args, skills, gradable, install=False, - config_dir=Path(bare_dir) if host else None, run_key=run_key, + config_dir=Path(bare_dir) if host else None, ) comparisons = staleness.pair(with_skill, without) @@ -1703,51 +1510,6 @@ def build_parser() -> argparse.ArgumentParser: "rate limits will take. Default: 0, inspect's own default." ), ) - staleness_parser.add_argument( - "--from-logs", - metavar="DIR", - help=( - "Rebuild the comparison from logs already on disk instead of " - "running anything. Both halves carry the skill, which side they " - "were, and the run that wrote them in their eval metadata, so a " - "log directory is enough to pair them -- and to tell this run's " - "halves from a neighbouring run of the same skill. Costs nothing " - "and reaches no model. Logs carrying no run id are passed over; " - "see --unlabelled-logs." - ), - ) - staleness_parser.add_argument( - "--unlabelled-logs", - action="store_true", - help=( - "With --from-logs, also read logs that carry no run id, which " - "means logs written before this command stamped one. Off by " - "default: an ordinary `behavioral` log is shaped exactly like a " - "pre-id graded half, so reading them unasked inflates the counts " - "with skills the run never covered, and can hand a comparison a " - "'with skill' half from a different run. Passing this asserts the " - "directory holds nothing else for these skills. A group still " - "needs a baseline side, and a side holding several logs is " - "reported `ungraded` rather than guessed at." - ), - ) - staleness_parser.add_argument( - "--run-id", - default="", - metavar="ID", - help=( - "With --from-logs, rebuild a named run rather than the most " - "recent. A log directory accumulates every staleness run done in " - "it, and a rebuild reads one per skill: reading all of them would " - "report the same case once per run and count it that many times " - "in the totals. The report names the run it read and lists the " - "ids it passed over, which is where this value comes from. An id " - "belongs to the invocation that minted it and covers whichever " - "skills that invocation was asked for, so naming one can leave a " - "skill in the directory unreported; when it does, the command " - "says which." - ), - ) staleness_parser.set_defaults(handler=cmd_staleness) select_parser = commands.add_parser( diff --git a/skillscope/engine/behavioral.py b/skillscope/engine/behavioral.py index 3837a5f..b6a2103 100644 --- a/skillscope/engine/behavioral.py +++ b/skillscope/engine/behavioral.py @@ -86,7 +86,6 @@ def build_task( *, solver_factory, baseline: bool = False, - run_key: str | None = None, ): """One inspect `Task` per skill: its cases, its skill installed, its scorer. @@ -123,7 +122,6 @@ def build_task( SKILL_KEY: skill, # Resolved here, where it is known to be right. See `SKILL_DIR_KEY`. SKILL_DIR_KEY: str(Path(skill_dir).resolve()), - **({STALENESS_RUN_KEY: run_key} if run_key else {}), }, tags=["skillscope", "baseline" if baseline else "graded"], dataset=samples, @@ -482,19 +480,6 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: BASELINE_KEY = "skillscope_baseline" SKILL_KEY = "skillscope_skill" -# Which staleness invocation wrote this log. Only `cmd_staleness` sets it, and -# it mints one value per invocation, so two logs carrying the same value are -# the two halves of one comparison and nothing else is. -# -# `--from-logs` needs this and cannot be built without it. The log directory is -# shared by `behavioral`, `verify` and `routing`, is gitignored wholesale and -# is never pruned, and every ordinary `behavioral` run stamps -# `BASELINE_KEY: False` -- so grouping on `(skill, baseline)` alone splices an -# unrelated month-old run in as the "with skill" half, under a table that reads -# as confidently as a real one. `run_eval` reached the same conclusion for the -# same directory and solved it the same way, with `CALL_TOKEN`. -STALENESS_RUN_KEY = "skillscope_staleness_run" - # Where the skill was on the machine that ran the agent. # # `_skill_marks` needs an absolute directory to recognise a tool call that @@ -596,7 +581,6 @@ def run( solver_factory, log_dir: str | None = None, baseline: bool = False, - run_key: str | None = None, attempts: int = 1, concurrency: int = 0, ) -> list[BehaviorOutcome]: @@ -606,10 +590,6 @@ def run( It does not itself remove the skill: the caller's `solver_factory` does that, and this says so in the output. - `run_key` is passed through to the task's metadata. See - `STALENESS_RUN_KEY`: only a staleness run sets one, and it is what lets - `--from-logs` pair two halves without pairing two unrelated runs. - `attempts` runs every case that many times, as inspect epochs, and the epoch comes back on each sample as `BehaviorOutcome.attempt`. One attempt is the ordinary run and leaves the logs exactly as they were. @@ -649,7 +629,6 @@ def run( model, solver_factory=solver_factory, baseline=baseline, - run_key=run_key, ), model=model, model_args=models.model_args(model), diff --git a/skillscope/engine/verify.py b/skillscope/engine/verify.py index 0061407..9e99652 100644 --- a/skillscope/engine/verify.py +++ b/skillscope/engine/verify.py @@ -219,7 +219,6 @@ def build_task( ctx: dict | None = None, effort: str | None = None, install: bool = True, - run_key: str | None = None, ): """One task per skill, solved by real Claude Code rather than our agent. @@ -257,11 +256,6 @@ def build_task( # See `behavioral.SKILL_DIR_KEY`: resolved on the machine that ran # the agent, because a reader cannot work it out afterwards. behavioral.SKILL_DIR_KEY: str(Path(skill_dir).resolve()), - # See `behavioral.STALENESS_RUN_KEY`: set only by a staleness run, - # and only there because only there are two logs meant to pair. - **( - {behavioral.STALENESS_RUN_KEY: run_key} if run_key else {} - ), }, tags=["skillscope", "graded" if install else "baseline"], setup=hooks.setup_solver(hook, skill, cases), @@ -300,15 +294,11 @@ def run( effort: str, *, install: bool = True, - run_key: str | None = None, attempts: int = 1, concurrency: int = 0, ) -> list[BehaviorOutcome]: """Mirrors `behavior.run`, so the CLI and the benchmark treat it the same. - `run_key` is stamped on the task, same as the other engine's. See - `behavioral.STALENESS_RUN_KEY`. - `attempts` and `concurrency` are inspect's epochs and sample limit, as in `behavioral.run`. Both legs take them, so a staleness run measures the same way whichever engine it asked for -- an option that worked on one @@ -341,7 +331,7 @@ def run( inspect_eval, build_task( skill, skill_cases, model, effort=effort, - install=install, run_key=run_key, + install=install, ), model=model, model_args=models.model_args(model), diff --git a/skillscope/staleness.py b/skillscope/staleness.py index 1c8776b..358c523 100644 --- a/skillscope/staleness.py +++ b/skillscope/staleness.py @@ -69,7 +69,7 @@ from __future__ import annotations -from dataclasses import asdict, dataclass, field, replace +from dataclasses import asdict, dataclass, field from .behavior import BehaviorOutcome @@ -112,12 +112,6 @@ class Comparison: id: str skill: str verdict: str - # Which staleness run produced this comparison. Empty on a live run, - # where there is only one and the report is about it. A rebuild sets it, - # because a log directory holds every run anyone has ever done in it and - # a report that does not say which one it read is not a measurement of - # anything in particular. See `from_logs`. - run: str = "" # Attempts that produced a grade, and how many of them met every positive # claim. One attempt each is the default; `passes_with` is then 0 or 1 and # the verdict rules read exactly as they did before repeats existed. @@ -460,286 +454,6 @@ def pair( return comparisons -def outcomes_from_log(log) -> tuple[str, bool, str | None, list[BehaviorOutcome]]: - """`(skill, was_baseline, run_key, outcomes)` for one inspect log. - - Rebuilt from the samples rather than from a dataset, because an offline - reader has no `Case` list: the log is all there is. Prompts are left empty - for the same reason -- the comparison is about verdicts, and inventing a - prompt field nobody wrote would be worse than an honest blank. - """ - from . import routing as routing_core - from .engine import behavioral, scorers - from .engine import routing as routing_engine - - eval_meta = log.eval.metadata or {} - skill = eval_meta.get(behavioral.SKILL_KEY) or log.eval.task - # Where the skill was when the agent ran, not where it is now. Absent from - # logs written before the key existed, which makes the room unknown rather - # than clean unless this machine happens to have the skill. See - # `behavioral.SKILL_DIR_KEY`. - skill_dir = eval_meta.get(behavioral.SKILL_DIR_KEY) - outcomes: list[BehaviorOutcome] = [] - for sample in log.samples or []: - checks: list[dict] = [] - for score in (sample.scores or {}).values(): - checks.extend((score.metadata or {}).get(scorers.CHECKS, [])) - error = sample.error.message if sample.error is not None else None - if error is None and not checks: - # Exactly what the live path sets (`behavioral._outcomes`), and it - # has to be the same string for the same reason the detector is - # shared: a sample whose checks could not be read rebuilds as - # `checks=[]`, which makes `_met()` False, which makes the case - # `earns_it` off the back of a harness failure. Without this the - # rebuild is *more permissive than the run it rebuilds*, quietly - # restoring a verdict the live run had voided as `ungraded`. - error = "case has no behavioral assertions to grade" - outcomes.append( - BehaviorOutcome( - id=str(sample.id), - skill=skill, - prompt="", - passed=error is None and bool(checks) and all(c["passed"] for c in checks), - elapsed_s=round(getattr(sample, "total_time", None) or 0.0, 2), - checks=checks, - error=error, - # inspect's own repeat counter. A run that asked for one - # attempt writes 1 on every sample, so this is the identity - # on everything but an `--attempts` run. - attempt=int(getattr(sample, "epoch", 1) or 1), - # Exactly what the live path sets, for the same reason as the - # error string above: a rebuild that left this False would - # report a provider outage as a skill that failed, which is - # the rebuild being more permissive than the run it rebuilds. - degraded=routing_core.is_provider_error(error), - # Both detected here too. Without the first, a rebuilt - # comparison reports every `earns_it` as never having fired; - # without the second it is *more permissive than the run it - # rebuilds*, quietly restoring verdicts the live run voided. - # - # `skill_dir` is what keeps the second true off this machine: - # the detector needs an absolute path to recognise, and left - # to work one out it works out this reader's rather than the - # run's. See `behavioral.SKILL_DIR_KEY`. - **dict( - zip( - ("activated", "contaminated"), - behavioral._inspect_room( - routing_engine, sample, skill, skill_dir - ), - ) - ), - ) - ) - return ( - skill, - bool(eval_meta.get(behavioral.BASELINE_KEY)), - eval_meta.get(behavioral.STALENESS_RUN_KEY), - outcomes, - ) - - -@dataclass -class Rebuild: - """The comparisons a log directory yielded, and the runs behind them. - - `runs` holds every staleness run the directory had for each skill, newest - first, whether or not it was used. Nothing prunes `.skillscope/logs`, so a - directory accumulates runs and "which one is this" stops being a detail: - the caller needs it to name the run in the report, and to say what a - `--run-id` could have asked for instead. - """ - - comparisons: list[Comparison] - runs: dict[str, list[str]] = field(default_factory=dict) - - -def from_logs( - log_dir: str, *, unlabelled: bool = False, run_id: str | None = None -) -> Rebuild: - """Rebuild the comparison from logs already on disk. - - Two runs become a pair because they carry the same skill, the same - `STALENESS_RUN_KEY`, and opposite `BASELINE_KEY` -- which is why all three - are written as metadata rather than inferred from a task name. - - Grouping on the run id is what makes this safe to point at - `.skillscope/logs`, which is the documented thing to do with it. That - directory is shared by `behavioral`, `verify` and `routing`, is gitignored - wholesale, and nothing prunes it; and every ordinary `behavioral` run - stamps `BASELINE_KEY: False`, so a run id is the only thing separating - this run's graded half from an unrelated run of the same skill last month. - - One run per skill, the newest, unless `run_id` names another. The run id - stops two runs being *spliced* into one comparison; it does not stop them - being *pooled*, and pooling is what a directory of accumulated runs - produces. Every run of a one-case skill used to add another copy of that - case to the report, so the second staleness run anybody did turned a - one-case suite into a two-case one and the headline counted it twice -- - under exactly the invocation the docs give for this. The runs passed over - are returned alongside, because a report that drops evidence silently is - the other half of the same fault. - - Newest is read off each log's own `created` stamp rather than the file's - mtime, which says when the directory was last copied about as often as it - says when the run happened. - - Logs with no run id are skipped unless `unlabelled` asks for them. See - `LEGACY_RUN` for what that costs and what it is for. They compete for - "newest" like any other run once asked for, and stay subject to the - ambiguity guard if they win. - - A directory holding only one side of a run is reported as a comparison - against nothing, the same way a case missing from one side is: silently - dropping it would read as a smaller, cleaner run. - - Logs that are not skillscope's own are passed over. A shared log directory - is normal, and failing on a neighbour's file would make this unusable in - exactly the directories it is meant for. - """ - from inspect_ai.log import list_eval_logs, read_eval_log - - from .engine import behavioral - - sides: dict[tuple[str, str], dict[bool, list[BehaviorOutcome]]] = {} - logs_per_side: dict[tuple[str, str], dict[bool, int]] = {} - written: dict[tuple[str, str], str] = {} - for info in list_eval_logs(log_dir): - log = read_eval_log(info.name) - if behavioral.BASELINE_KEY not in (log.eval.metadata or {}): - continue - skill, baseline, run_key, outcomes = outcomes_from_log(log) - key = (skill, run_key or LEGACY_RUN) - sides.setdefault(key, {}).setdefault(baseline, []).extend(outcomes) - tally = logs_per_side.setdefault(key, {}) - tally[baseline] = tally.get(baseline, 0) + 1 - written[key] = max(written.get(key, ""), str(log.eval.created or "")) - - candidates = [ - key - for key in sorted(sides) - if key[1] != LEGACY_RUN or (unlabelled and logs_per_side[key].get(True)) - ] - # Newest first, with a labelled run always ahead of the unlabelled - # bucket. `LEGACY_RUN` is a bucket and not a run -- it is every log that - # carried no id -- so letting it win on a timestamp would pick a pile over - # a measurement. The run id breaks the remaining ties, so two runs written - # in the same second order the same way on every machine. - runs: dict[str, list[str]] = {} - for key in sorted( - candidates, - key=lambda k: (k[1] != LEGACY_RUN, written.get(k, ""), k[1]), - reverse=True, - ): - runs.setdefault(key[0], []).append(key[1]) - chosen = [ - key - for key in candidates - if key[1] == (runs[key[0]][0] if run_id is None else run_id) - ] - - comparisons: list[Comparison] = [] - for key in sorted(chosen): - half = sides[key] - counts = logs_per_side[key] - ambiguous = key[1] == LEGACY_RUN and any(n > 1 for n in counts.values()) - comparisons.extend( - replace(comparison, run=key[1]) - for comparison in pair( - _no_duplicates(half.get(False, []), ambiguous), - _no_duplicates(half.get(True, []), ambiguous), - ) - ) - return Rebuild(comparisons=comparisons, runs=runs) - - -# The bucket for logs written before a run id was stamped on them. -# -# Skipped by default, and taken only when the caller asks with -# `--unlabelled-logs`. Both halves of that rule are load-bearing. -# -# Taking them by default was wrong, and not only for old directories. A plain -# `skillscope behavioral` run stamps `BASELINE_KEY: False` and never stamps a -# run id, because only `cmd_staleness` mints one -- so an ordinary behavioral -# log is shaped exactly like a pre-key graded half and always will be. Pooling -# them inflated `cases` and `ungraded` with every skill the user had ever run -# `behavioral` against, and could silently supply the "with skill" side of a -# comparison it was never part of. That is the splicing bug this module's run -# id exists to prevent, arriving through the back door. -# -# Refusing them outright was wrong too, and the reason outlives the migration -# that prompted it. A log directory is a record somebody already paid an agent -# run for; a reader that cannot open one because of a metadata key added after -# it was written has made that run worthless to settle a question it holds the -# answer to. The population is small and does not grow -- every run since the -# key arrived labels itself -- but "small" is the argument for a flag rather -# than for a refusal. -# -# So the caller asserts it. Even then a group needs a baseline side, which -# nothing but `cmd_staleness` ever writes, so a directory of ordinary -# `behavioral` logs stays out whatever flags it is given. And where a side -# holds more than one log there is a choice to make and no way to make it, so -# every case in that group is `ungraded` rather than guessed at. -# -# A labelled run never falls in here, so none of this can weaken a run written -# by this version. -LEGACY_RUN = "unlabelled" - - -def _no_duplicates( - outcomes: list[BehaviorOutcome], ambiguous: bool = False -) -> list[BehaviorOutcome]: - """`outcomes`, with every case this side cannot answer for errored. - - `ambiguous` errors all of them: see `LEGACY_RUN`, where unlabelled logs - leave several candidates for a side and no way to tell which run each - belongs to. Otherwise only a case this side answers twice *for the same - attempt* is errored. - - One run key and one side can still span several logs -- a run grades one - skill per task, and a run key covers the whole invocation -- and nothing - stops two of them carrying the same case id. `pair` groups a side's - attempts by case id, so without this two logs answering the same case - would read as one case attempted twice, and a collision between runs would - arrive disguised as the evidence `--attempts` exists to produce. - - The attempt number is what tells a repeat from a collision. A side that - ran three attempts answers each case three times by design, once per - attempt; two logs answering the same case *and* the same attempt is the - ambiguity, and no legitimate run produces one. - - Both copies are errored, not just the later one: which of them is this - side's answer is exactly what cannot be told, so keeping either would be - the guess. `classify` reads an errored attempt as one that measured - nothing, consistent with how this module reports every other ambiguity it - cannot resolve. - """ - if ambiguous: - return [ - replace( - outcome, - error="these logs carry no run id and there is more than one " - "of them on a side, so which run this case belongs to cannot " - "be told apart", - ) - for outcome in outcomes - ] - tally: dict[tuple[str, int], int] = {} - for outcome in outcomes: - key = (outcome.id, outcome.attempt) - tally[key] = tally.get(key, 0) + 1 - return [ - outcome if tally[(outcome.id, outcome.attempt)] == 1 - else replace( - outcome, - error="this run has more than one log reporting this case on the " - "same attempt, and which of them is its answer cannot be told " - "apart", - ) - for outcome in outcomes - ] - - def summarize(comparisons: list[Comparison], meta: dict) -> dict: from .engine.behavioral import UNKNOWN_ROOM @@ -921,11 +635,9 @@ def render_markdown(summary: dict) -> str: else "**Nothing was graded both ways**" ) - # Said only when it is known. A `--from-logs` rebuild runs no model, and - # the invocation's own `--model`/`--effort` defaults describe the command - # that read the logs rather than the runs that wrote them -- rebuilding a - # `sonnet`/`low` run with a bare `--from-logs` printed "on `opus` (effort - # `high`)", which is not a hedge but a false statement about a measurement. + # Still conditional rather than assumed: `summarize` is given whatever + # meta its caller built, and a report that names a model it was not told + # about states a measurement that was never taken. ran_on = ( f" on `{meta['model']}` (effort `{meta['effort']}`)" if meta.get("model") and meta.get("effort") @@ -935,7 +647,7 @@ def render_markdown(summary: dict) -> str: # `0/3` and `1/1` against `0/1` are the same verdict and not the same # evidence, and a reader who does not know which they are holding cannot # tell them apart. Read off the cases rather than the invocation's own - # `--attempts`, so a `--from-logs` rebuild describes the runs it read. + # `--attempts`, so the report describes the run rather than the request. per_side = {c["attempts_with"] for c in summary["cases"]} | { c["attempts_without"] for c in summary["cases"] } @@ -985,63 +697,6 @@ def render_markdown(summary: dict) -> str: "the cases it did not run.", "", ] - if meta.get("from_logs"): - lines += [ - f"Rebuilt offline from the logs in `{meta['from_logs']}`; nothing " - "was run. The model and effort are whatever wrote them, which the " - "logs do not have to agree on, so they are not claimed here.", - "", - ] - # Which run this is a report about. A log directory accumulates every - # staleness run anybody has done in it, so naming the one read is the - # difference between a measurement and a pile of them -- see - # `from_logs` on what reading the pile used to do to the totals. - used = {c["skill"]: c["run"] for c in summary["cases"] if c["run"]} - if used: - lines += [ - "Read from " - + ", ".join( - f"run `{run}` for `{skill}`" - for skill, run in sorted(used.items()) - ) - + ".", - "", - ] - # A run id belongs to an invocation and not to a skill: a staleness - # run mints one id and stamps it on every skill it covers. So "was - # this run read" is a question about the whole report rather than - # about one skill's row -- asked per skill, a run read for `beta` was - # announced as unread by `alpha`, two lines under the sentence naming - # it as read. - # - # Listed per skill even so, because that is the only form in which - # the ids are actionable: `--run-id` is global, and which skills an - # id covers is whichever its own invocation was asked for. - read = set(used.values()) - unread = { - skill: [run for run in ids if run not in read] - for skill, ids in (meta.get("runs") or {}).items() - } - unread = {skill: ids for skill, ids in unread.items() if ids} - spare = {run for ids in unread.values() for run in ids} - if used and spare: - shown = "; ".join( - ", ".join(f"`{run}`" for run in ids[:3]) - + (f" and {len(ids) - 3} more" if len(ids) > 3 else "") - + f" for `{skill}`" - for skill, ids in sorted(unread.items()) - ) - lines += [ - f"> **{len(spare)} other staleness run(s) in this directory " - "were not read.** One run per skill is read" - + ("" if meta.get("run_id") else ", the most recent") - + ", because reading several reports the same case once per " - f"run: {shown}. `--run-id` reads one of them instead. An id " - "covers whichever skills its own run was asked for, which " - "need not be every skill here.", - "", - ] - # Nothing moved at all. The row `earns_it | 0` is true and says nothing, # and this is the one result the command exists to surface, so it is said # rather than left to be noticed in a table. diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 13b4327..73b18b5 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -6306,12 +6306,11 @@ def _staleness_harness(test, *, dataset=None): calls: list[dict] = [] - def record(args, skills, gradable, *, install=True, config_dir=None, run_key=None): + def record(args, skills, gradable, *, install=True, config_dir=None): calls.append( { "install": install, "config_dir": config_dir, - "run_key": run_key, "cases": [c.id for c in gradable], } ) @@ -6443,20 +6442,6 @@ def __init__(self, sample_id, checks, error=None, epoch=1): self.epoch = epoch -class _StubLog: - def __init__(self, skill, baseline, samples, run_key="run-1", created="2026-01-01"): - metadata = { - engine_behavioral.SKILL_KEY: skill, - engine_behavioral.BASELINE_KEY: baseline, - } - if run_key is not None: - metadata[engine_behavioral.STALENESS_RUN_KEY] = run_key - self.eval = types.SimpleNamespace( - task=f"behavioral-{skill}", metadata=metadata, created=created - ) - self.samples = samples - - class TestGuardRailsAreNotJudgedAsIfTheyDiscriminated(unittest.TestCase): """A guard rail holds with or without the skill -- that is its job. @@ -6939,26 +6924,12 @@ def _run(self, *argv): cli.cmd_staleness(parser.parse_args(["staleness", *argv])) return str(caught.exception) - def test_attempts_cannot_change_a_rebuild(self) -> None: - self.assertIn("--attempts", self._run("--from-logs", "d", "--attempts", "3")) - - def test_the_cap_cannot_change_a_rebuild(self) -> None: - self.assertIn("--max-cases", self._run("--from-logs", "d", "--max-cases", "2")) - def test_a_run_of_no_attempts_is_refused(self) -> None: self.assertIn("would run nothing", self._run("--attempts", "0")) def test_a_negative_count_is_refused(self) -> None: self.assertIn("cannot be negative", self._run("--concurrency", "-1")) - def test_choosing_the_evals_cannot_change_a_rebuild(self) -> None: - # Which evals were graded is a property of the run that wrote the - # logs, like the attempts and the cap beside it. It used to be - # accepted and ignored. - self.assertIn( - "--no-extended", self._run("--from-logs", "d", "--no-extended") - ) - def test_a_harness_with_no_staleness_leg_is_refused(self) -> None: # The baseline half is only a measurement if the room is empty, and # the check that establishes that reads Claude Code's credentials and @@ -6974,38 +6945,12 @@ def test_the_environment_can_name_that_harness_too(self) -> None: def test_the_default_harness_is_not_refused(self) -> None: # The refusal is about the room check, not about the flag. Naming the # default explicitly has to behave exactly like not naming it. - said = self._run("--harness", "claude-code", "--from-logs", "/nope") - self.assertNotIn("no staleness leg", said) - - def test_naming_the_agent_cannot_change_a_rebuild(self) -> None: - # A rebuild runs no agent, so which one it would have run is a - # property of the logs. Refused rather than ignored, like the rest. - said = self._run("--from-logs", "d", "--harness", "copilot") - self.assertIn("--harness", said) - self.assertIn("cannot change a --from-logs rebuild", said) - - def test_an_ambient_harness_does_not_refuse_a_rebuild(self) -> None: - # The variable is the shell's, not this invocation's argument. - # Refusing a free rebuild over it would be a refusal nobody asked for. - with mock.patch.dict(os.environ, {"SKILLSCOPE_HARNESS": "copilot"}): - said = self._run("--from-logs", "/nope") - self.assertNotIn("cannot change a --from-logs rebuild", said) - - def test_a_run_id_means_nothing_without_a_rebuild(self) -> None: - # A live run mints its own id and stamps it on both halves, so there - # is nothing here to pick between. - self.assertIn("--from-logs", self._run("--run-id", "abc")) - - -class TestASideCanBeAttemptedMoreThanOnce(unittest.TestCase): - """`--attempts` is what turns a verdict from an anecdote into evidence. - - One run against one run is a coin flip against a coin flip: an agent - disagrees with itself between runs, so a case can pass with the skill and - fail without it for no reason the skill is responsible for. The verdict - rule is unchanged -- a side passed if any attempt met every claim -- so - what the repeats add is the counts beside it. - """ + # + # This used to pass `--from-logs`, which returned before the guard + # was ever reached -- so it asserted the absence of a refusal that + # could not have fired, and would have gone on passing had the guard + # been written to refuse the default too. + self.assertNotIn("no staleness leg", self._run("--harness", "claude-code")) def _pair(self, with_results, without_results): return staleness.pair( @@ -7244,86 +7189,6 @@ def test_a_clean_run_says_nothing_about_firing(self) -> None: self.assertNotIn("never fired on the attempt", report) -class TestAComparisonCanBeRebuiltFromLogs(unittest.TestCase): - """The pairing is written into the logs, so a directory is enough. - - Without this the two runs are related only in the memory of the process - that launched them, and a finished run cannot be looked at again without - paying for it twice over. - """ - - def test_a_log_reports_which_side_it_was(self) -> None: - log = _StubLog("alpha", True, [_StubSample("c", [_check("do it", False)])]) - skill, baseline, _, outcomes = staleness.outcomes_from_log(log) - self.assertEqual(skill, "alpha") - self.assertTrue(baseline) - self.assertEqual(len(outcomes), 1) - self.assertFalse(outcomes[0].passed) - - def test_a_sample_passes_only_when_every_check_did(self) -> None: - log = _StubLog("alpha", False, [ - _StubSample("c", [_check("a", True), _check("b", True)]), - _StubSample("d", [_check("a", True), _check("b", False)]), - ]) - _, _, _, outcomes = staleness.outcomes_from_log(log) - self.assertEqual([o.passed for o in outcomes], [True, False]) - - def test_a_sample_with_no_checks_has_not_passed(self) -> None: - # An ungraded sample is not a green one. - log = _StubLog("alpha", False, [_StubSample("c", [])]) - _, _, _, outcomes = staleness.outcomes_from_log(log) - self.assertFalse(outcomes[0].passed) - - def test_a_rebuild_is_no_more_permissive_than_the_run_it_rebuilds(self) -> None: - # A sample whose checks could not be read is ungraded on the live path - # (`behavioral._outcomes` sets an error for it). Rebuilt without that, - # it comes back as `checks=[]`, which makes `_met()` False, which reads - # as the baseline having failed -- so a harness failure is promoted to - # `earns_it` and inflates the headline number. - # - # Asserted on the verdict, not on the source. The version of this that - # grepped `outcomes_from_log` for "_inspect_room" and "contaminated" - # passed for the whole life of the defect it was named after. - graded = _StubLog("alpha", False, [_StubSample("c", [_check("do it", True)])]) - unreadable = _StubLog("alpha", True, [_StubSample("c", [])]) - _, _, _, with_skill = staleness.outcomes_from_log(graded) - _, _, _, without = staleness.outcomes_from_log(unreadable) - - self.assertEqual( - without[0].error, "case has no behavioral assertions to grade" - ) - self.assertEqual(staleness.classify([with_skill[0]], [without[0]]), "ungraded") - - def test_the_live_path_and_the_rebuild_word_it_the_same_way(self) -> None: - # Two strings that have to match for the two paths to agree, and - # nothing else makes them. - live = inspect.getsource(engine_behavioral._outcomes) - rebuilt = inspect.getsource(staleness.outcomes_from_log) - message = "case has no behavioral assertions to grade" - self.assertIn(message, live) - self.assertIn(message, rebuilt) - - def test_the_two_sides_pair_on_the_skill_they_name(self) -> None: - graded = _StubLog("alpha", False, [_StubSample("c", [_check("do it", True)])]) - baseline = _StubLog("alpha", True, [_StubSample("c", [_check("do it", False)])]) - sides = {} - for log in (graded, baseline): - skill, is_baseline, _, outcomes = staleness.outcomes_from_log(log) - sides.setdefault(skill, {}).setdefault(is_baseline, []).extend(outcomes) - comparisons = staleness.pair(sides["alpha"][False], sides["alpha"][True]) - self.assertEqual(len(comparisons), 1) - # Both sides found each other: one attempt graded on each, and the - # case moved. `unattributed` rather than `earns_it` because a stub log - # carries no tool calls, so nothing in it shows the skill firing -- - # which is the honest reading of a rebuild with no activation - # evidence, and the one `classify` gives. - self.assertEqual(comparisons[0].attempts_with, 1) - self.assertEqual(comparisons[0].attempts_without, 1) - self.assertEqual(comparisons[0].passes_with, 1) - self.assertEqual(comparisons[0].passes_without, 0) - self.assertEqual(comparisons[0].verdict, "unattributed") - - class TestTheBaselineRunIsTheSameRunWithoutTheSkill(unittest.TestCase): """The flag has to reach the driver, or the two halves are not comparable.""" @@ -7761,288 +7626,12 @@ def test_both_halves_grade_the_same_cases(self) -> None: self.assertEqual(harness.calls[0]["cases"], harness.calls[1]["cases"]) self.assertTrue(harness.calls[0]["cases"]) - def test_both_halves_carry_one_run_key(self) -> None: - # What `--from-logs` pairs on. Two different keys leave the halves - # unpairable; no key leaves them pairable with anything. - with _staleness_harness(self) as harness: - harness.run() - keys = {c["run_key"] for c in harness.calls} - self.assertEqual(len(keys), 1) - self.assertTrue(keys.pop()) - - def test_two_invocations_do_not_share_a_run_key(self) -> None: - with _staleness_harness(self) as harness: - harness.run() - harness.run() - self.assertNotEqual(harness.calls[0]["run_key"], harness.calls[2]["run_key"]) - def test_a_dataset_with_nothing_to_grade_runs_neither_half(self) -> None: with _staleness_harness(self, dataset=tier0_dataset("alpha")) as harness: self.assertEqual(harness.run(), 0) self.assertEqual(harness.calls, []) -class TestARebuildPairsOneRunAndNotItsNeighbours(unittest.TestCase): - """`.skillscope/logs` is shared, never pruned, and gitignored wholesale. - - Every ordinary `behavioral` run stamps `BASELINE_KEY: False`, so grouping - on `(skill, baseline)` alone made an unrelated run of the same skill a - candidate for this comparison's "with skill" half -- rendered into a table - that reads exactly as confidently as a real one. `docs/usage.md` documents - pointing `--from-logs` at that very directory. - """ - - def _rebuild(self, logs, unlabelled=True, run_id=None): - return self._read(logs, unlabelled, run_id).comparisons - - def _read(self, logs, unlabelled=True, run_id=None): - info = [types.SimpleNamespace(name=f"log-{i}") for i in range(len(logs))] - by_name = {i.name: log for i, log in zip(info, logs)} - with mock.patch("inspect_ai.log.list_eval_logs", lambda d: info), \ - mock.patch("inspect_ai.log.read_eval_log", lambda n: by_name[n]): - return staleness.from_logs( - "anywhere", unlabelled=unlabelled, run_id=run_id - ) - - def _log(self, baseline, passed, run_key="run-1", case="c", created="2026-01-01"): - return _StubLog( - "alpha", baseline, - [_StubSample(case, [_check("do it", passed)])], - run_key=run_key, - created=created, - ) - - def test_one_runs_two_halves_pair(self) -> None: - comparisons = self._rebuild([self._log(False, True), self._log(True, False)]) - # One case, graded on both sides, that passed only with the skill. A - # stub log has no tool calls to show the skill firing, so the verdict - # is `unattributed`; what this is about is that the two halves found - # each other at all. - self.assertEqual( - [(c.passes_with, c.passes_without, c.verdict) for c in comparisons], - [(1, 0, "unattributed")], - ) - - def test_a_neighbouring_runs_log_is_not_spliced_in(self) -> None: - # The stale graded half says the case passed; this run's says it - # failed. Pooled, the pair reads `earns_it` off a run nobody asked - # about. The newest run is read, and only its own halves reach it. - rebuild = self._read([ - self._log(False, True, run_key="old", created="2026-01-01"), - self._log(False, False, run_key="new", created="2026-02-02"), - self._log(True, False, run_key="new", created="2026-02-02"), - ]) - self.assertEqual( - [(c.run, c.verdict) for c in rebuild.comparisons], - [("new", "fails_either_way")], - ) - self.assertEqual(rebuild.runs, {"alpha": ["new", "old"]}) - - def _two_runs(self): - return [ - self._log(False, True, run_key="first", created="2026-01-01"), - self._log(True, False, run_key="first", created="2026-01-01"), - self._log(False, True, run_key="second", created="2026-03-03"), - self._log(True, False, run_key="second", created="2026-03-03"), - ] - - def test_two_runs_of_one_skill_are_not_both_reported(self) -> None: - # The run id stopped two runs being *spliced*; it did not stop them - # being *pooled*. A one-case skill came back as a two-case suite with - # the same row twice, on the second staleness run anybody did, under - # the invocation `docs/usage.md` gives. - rebuild = self._read(self._two_runs()) - self.assertEqual([c.run for c in rebuild.comparisons], ["second"]) - self.assertEqual(rebuild.runs, {"alpha": ["second", "first"]}) - - def _across_skills(self, *specs): - return [ - _StubLog(skill, baseline, - [_StubSample(f"c-{skill}", [_check("do it", passed)])], - run_key=run, created=created) - for skill, baseline, passed, run, created in specs - ] - - def test_a_run_read_for_one_skill_is_not_called_unread_by_another(self) -> None: - # One invocation mints one run id and stamps it on every skill it - # covers, so a run id spans skills. Asked per skill, `RUN1` was read - # for `beta` and announced as unread by `alpha`, two lines under the - # sentence naming it as read. - rebuild = self._read(self._across_skills( - ("alpha", False, True, "RUN1", "2026-01-01"), - ("alpha", True, False, "RUN1", "2026-01-01"), - ("beta", False, True, "RUN1", "2026-01-01"), - ("beta", True, False, "RUN1", "2026-01-01"), - ("alpha", False, True, "RUN2", "2026-03-03"), - ("alpha", True, False, "RUN2", "2026-03-03"), - )) - report = staleness.render_markdown(staleness.summarize( - rebuild.comparisons, {"from_logs": "/l", "runs": rebuild.runs}, - )) - self.assertIn("run `RUN2` for `alpha`", report) - self.assertIn("run `RUN1` for `beta`", report) - self.assertNotIn("were not read", report) - - def test_an_unread_run_is_listed_with_the_skill_it_belongs_to(self) -> None: - # `--run-id` is global, so an id is only actionable beside the skill - # whose run it was. - rebuild = self._read(self._across_skills( - ("alpha", False, True, "A1", "2026-01-01"), - ("alpha", True, False, "A1", "2026-01-01"), - ("alpha", False, True, "A2", "2026-03-03"), - ("alpha", True, False, "A2", "2026-03-03"), - ("beta", False, True, "B1", "2026-02-02"), - ("beta", True, False, "B1", "2026-02-02"), - )) - report = staleness.render_markdown(staleness.summarize( - rebuild.comparisons, {"from_logs": "/l", "runs": rebuild.runs}, - )) - self.assertIn("1 other staleness run(s)", report) - self.assertIn("`A1` for `alpha`", report) - - def test_an_older_run_can_be_asked_for_by_id(self) -> None: - rebuild = self._read(self._two_runs(), run_id="first") - self.assertEqual([c.run for c in rebuild.comparisons], ["first"]) - - def test_an_id_that_names_no_run_reads_nothing_and_says_what_there_is(self) -> None: - rebuild = self._read(self._two_runs(), run_id="third") - self.assertEqual(rebuild.comparisons, []) - self.assertEqual(rebuild.runs, {"alpha": ["second", "first"]}) - - def test_the_report_names_the_run_it_read_and_what_it_passed_over(self) -> None: - rebuild = self._read(self._two_runs()) - report = staleness.render_markdown(staleness.summarize( - rebuild.comparisons, {"from_logs": "/l", "runs": rebuild.runs}, - )) - self.assertIn("run `second` for `alpha`", report) - self.assertIn("1 other staleness run(s)", report) - self.assertIn("`first`", report) - - def test_unlabelled_logs_are_not_read_unless_asked_for(self) -> None: - # The default. A plain `behavioral` log carries `BASELINE_KEY: False` - # and no run id, which is the same shape a pre-id graded half has, so - # reading unlabelled logs unasked pools whatever is in the directory. - comparisons = self._rebuild([ - self._log(False, True, run_key=None), - self._log(True, False, run_key=None), - ], unlabelled=False) - self.assertEqual(comparisons, []) - - def test_an_unlabelled_pair_is_read_when_asked_for(self) -> None: - # What `--unlabelled-logs` is for: one log a side, nothing to choose - # between, and a baseline half that only a staleness run ever writes. - comparisons = self._rebuild([ - self._log(False, True, run_key=None), - self._log(True, False, run_key=None), - ]) - self.assertEqual( - [(c.passes_with, c.passes_without, c.verdict) for c in comparisons], - [(1, 0, "unattributed")], - ) - - def test_repeats_of_one_case_are_not_read_as_a_collision(self) -> None: - # What `--attempts` writes: one log a side, each case answered once - # per attempt. Keyed on the case id alone this looked exactly like two - # logs fighting over one case, and every case came back `ungraded`. - comparisons = self._rebuild([ - _StubLog("alpha", False, [ - _StubSample("c", [_check("do it", True)], epoch=1), - _StubSample("c", [_check("do it", True)], epoch=2), - ]), - _StubLog("alpha", True, [ - _StubSample("c", [_check("do it", False)], epoch=1), - _StubSample("c", [_check("do it", False)], epoch=2), - ]), - ]) - self.assertEqual( - [(c.passes_with, c.attempts_with, c.passes_without, c.attempts_without) - for c in comparisons], - [(2, 2, 0, 2)], - ) - - def test_two_logs_answering_the_same_attempt_are_still_refused(self) -> None: - # The ambiguity the guard exists for, which repeats must not hide: - # which of the two is this side's answer cannot be told, so neither - # is kept. - graded = _StubLog("alpha", False, [_StubSample("c", [_check("do it", True)])]) - comparisons = self._rebuild([ - graded, - _StubLog("alpha", False, [_StubSample("c", [_check("do it", True)])]), - _StubLog("alpha", True, [_StubSample("c", [_check("do it", False)])]), - ]) - self.assertEqual([c.verdict for c in comparisons], ["ungraded"]) - self.assertIn("same attempt", comparisons[0].error) - - def test_graded_only_unlabelled_logs_are_never_read(self) -> None: - # The common hole. `behavioral` and `verify` runs never carry a run - # id by design, so these keep arriving forever -- and pooled they - # inflated `cases` and `ungraded` with every skill the user had ever - # run `behavioral` against, plus a `per_skill` row for it. - # - # Asking for unlabelled logs does not buy this: a baseline half is - # the one thing nothing but `cmd_staleness` writes, so requiring one - # keeps ordinary behavioral logs out whatever flags are passed. - for asked in (True, False): - with self.subTest(unlabelled=asked): - comparisons = self._rebuild( - [self._log(False, True, run_key=None)], unlabelled=asked - ) - self.assertEqual(comparisons, []) - - def test_a_behavioral_log_cannot_supply_a_half_to_a_stale_baseline(self) -> None: - # The rarer hole, and the original splicing bug through the back door: - # a plain `behavioral` log for a skill, plus a leftover pre-id - # baseline for the same skill, is one log a side and was paired. - # `summarize` counts what comes back, so this has to stay empty. - comparisons = self._rebuild([ - self._log(False, True, run_key=None), # indistinguishable from - # a plain behavioral run - self._log(True, False, run_key=None), - ], unlabelled=False) - self.assertEqual(staleness.summarize(comparisons, {})["totals"]["cases"], 0) - - def test_unlabelled_logs_with_a_choice_to_make_are_not_guessed_at(self) -> None: - # Two candidates for the graded side and no way to tell which run - # either belongs to. That ambiguity is the whole hazard, so it is - # reported rather than silently resolved to the older log. - comparisons = self._rebuild([ - self._log(False, True, run_key=None), - self._log(False, False, run_key=None, case="d"), - self._log(True, False, run_key=None), - ]) - self.assertTrue(comparisons) - self.assertEqual({c.verdict for c in comparisons}, {"ungraded"}) - self.assertIn("no run id", comparisons[0].error) - - def test_a_labelled_run_never_falls_back_to_the_unlabelled_bucket(self) -> None: - # A labelled half and an unlabelled one are not two halves of - # anything, however tempting the arithmetic. The labelled run is read - # and reported as the lone half it is, rather than paired into an - # `earns_it`; `LEGACY_RUN` is a bucket of whatever carried no id, so - # it never outranks a run however recently its logs were written. - rebuild = self._read([ - self._log(False, True, run_key="new", created="2026-01-01"), - self._log(True, False, run_key=None, created="2026-09-09"), - ]) - self.assertEqual( - [(c.run, c.verdict) for c in rebuild.comparisons], [("new", "ungraded")] - ) - self.assertIn("no result from", rebuild.comparisons[0].error) - self.assertEqual(rebuild.runs, {"alpha": ["new", staleness.LEGACY_RUN]}) - - def test_two_logs_reporting_one_case_are_reported_rather_than_resolved(self) -> None: - # `pair` builds a dict keyed by case id, so a duplicate collapses - # silently -- and `list_eval_logs` returns newest first, which makes - # the survivor the older of the two. - comparisons = self._rebuild([ - self._log(False, True), - self._log(False, False), - self._log(True, False), - ]) - self.assertEqual([c.verdict for c in comparisons], ["ungraded"]) - self.assertIn("more than one log", comparisons[0].error) - - class TestAStalenessRunHonoursHooksTheSameWayBehavioralDoes(unittest.TestCase): """It drives the same `build_task`, so it cannot skip what that refuses to. @@ -8083,212 +7672,6 @@ def test_routing_is_still_exempt(self) -> None: self.assertIsNone(self._refused("routing")) -class TestAnOfflineRebuildDoesNotClaimAModelItNeverRan(unittest.TestCase): - """`--from-logs` reaches no model, so it has no model to report. - - The meta dict passed `args.model` / `args.effort` straight through, and - nothing was read off the logs -- so rebuilding a run taken on `sonnet` at - `low` with a bare `--from-logs` printed "on `opus` (effort `high`)". Not a - hedge: a false statement about where a measurement came from, in the one - mode that never runs `enforce_model_policy` either. - """ - - def _report(self, meta): - comparisons = staleness.pair( - [_checked("c", True, [_check("do it", True)])], - [_checked("c", False, [_check("do it", False)])], - ) - return staleness.render_markdown(staleness.summarize(comparisons, meta)) - - def test_a_rebuild_names_no_model(self) -> None: - report = self._report({"from_logs": "/logs", "skills": ["alpha"]}) - headline = report.split("\n\n")[1] - self.assertNotIn("(effort", headline) - self.assertNotIn(" on `", headline) - self.assertIn("Rebuilt offline", report) - self.assertIn("/logs", report) - - def test_a_live_run_still_names_its_model(self) -> None: - report = self._report({"model": "sonnet", "effort": "low"}) - self.assertIn("`sonnet`", report) - self.assertIn("effort `low`", report) - - def test_the_command_passes_no_model_through_on_a_rebuild(self) -> None: - captured: dict = {} - real = staleness.summarize - - def record(comparisons, meta): - captured.update(meta) - return real(comparisons, meta) - - args = cli.build_parser().parse_args( - ["staleness", "--from-logs", "/logs", "--model", "opus"] - ) - with mock.patch.object(staleness, "summarize", record), \ - mock.patch.object( - staleness, "from_logs", - lambda d, **k: staleness.Rebuild(staleness.pair( - [_checked("c", True, [_check("x", True)])], - [_checked("c", False, [_check("x", False)])], - )), - ), \ - mock.patch.object(cli, "_write_report", lambda *a, **k: None): - cli.cmd_staleness(args) - self.assertNotIn("model", captured) - self.assertNotIn("effort", captured) - self.assertEqual(captured["from_logs"], "/logs") - - -class TestUnlabelledLogsAreOptedIntoRatherThanAssumed(unittest.TestCase): - """The flag has to reach `from_logs`, and mean nothing without a rebuild.""" - - def _run(self, argv): - captured: dict = {} - - def record(log_dir, *, unlabelled=False, run_id=None): - captured["unlabelled"] = unlabelled - captured["run_id"] = run_id - return staleness.Rebuild( - staleness.pair( - [_checked("c", True, [_check("x", True)])], - [_checked("c", False, [_check("x", False)])], - ) - ) - - args = cli.build_parser().parse_args(argv) - with mock.patch.object(staleness, "from_logs", record), \ - mock.patch.object(cli, "_write_report", lambda *a, **k: None): - cli.cmd_staleness(args) - return captured - - def test_a_rebuild_does_not_read_unlabelled_logs_by_default(self) -> None: - self.assertIs(self._run(["staleness", "--from-logs", "/l"])["unlabelled"], False) - - def test_the_flag_reaches_the_rebuild(self) -> None: - self.assertIs( - self._run( - ["staleness", "--from-logs", "/l", "--unlabelled-logs"] - )["unlabelled"], - True, - ) - - def test_the_flag_is_refused_without_a_rebuild(self) -> None: - # A live run stamps a run id on both halves, so there is nothing - # unlabelled for this to reach. Refused rather than ignored. - args = cli.build_parser().parse_args(["staleness", "--unlabelled-logs"]) - with self.assertRaises(SystemExit) as caught: - cli.cmd_staleness(args) - self.assertIn("--from-logs", str(caught.exception)) - - -class TestARebuildIsNarrowedRatherThanIgnored(unittest.TestCase): - """`--skill` and `--only` were accepted and ignored under `--from-logs`. - - The command refuses `--attempts` with a rebuild on the reasoning that an - argument which cannot do anything should be refused rather than accepted. - These two can do something: a shared log directory holding several skills - is exactly where a reader wants to ask about one of them. - """ - - def _rebuilder(self): - def rebuild(log_dir, **kwargs): - return staleness.Rebuild(staleness.pair( - [_checked("a", True, [_check("x", True)], "alpha"), - _checked("b", True, [_check("x", True)], "beta")], - [_checked("a", False, [_check("x", False)], "alpha"), - _checked("b", False, [_check("x", False)], "beta")], - )) - return rebuild - - def _summary(self, *argv): - captured: dict = {} - args = cli.build_parser().parse_args( - ["staleness", "--from-logs", "/l", *argv] - ) - with mock.patch.object(staleness, "from_logs", self._rebuilder()), \ - mock.patch.object( - cli, "_write_report", - lambda summary, *a, **k: captured.update(summary), - ): - cli.cmd_staleness(args) - return captured - - def test_the_whole_directory_is_reported_when_nothing_narrows_it(self) -> None: - self.assertEqual(self._summary()["meta"]["skills"], ["alpha", "beta"]) - - def test_a_skill_narrows_what_is_reported(self) -> None: - self.assertEqual(self._summary("--skill", "alpha")["meta"]["skills"], ["alpha"]) - - def test_a_case_id_narrows_what_is_reported(self) -> None: - self.assertEqual( - [c["id"] for c in self._summary("--only", "b")["cases"]], ["b"] - ) - - def test_a_filter_matching_nothing_is_refused_rather_than_reported_empty(self) -> None: - args = cli.build_parser().parse_args( - ["staleness", "--from-logs", "/l", "--skill", "gamma"] - ) - with mock.patch.object(staleness, "from_logs", self._rebuilder()), \ - mock.patch.object(cli, "_write_report", lambda *a, **k: None), \ - self.assertRaises(SystemExit) as caught: - cli.cmd_staleness(args) - self.assertIn("gamma", str(caught.exception)) - - def test_a_run_id_that_misses_a_skill_says_which(self) -> None: - # `--run-id` is global and a run id is not: naming one drops every - # skill its invocation did not cover. The "no such run" guard cannot - # catch it, because it only fires when nothing matched at all. - def rebuild(log_dir, **kwargs): - return staleness.Rebuild( - staleness.pair( - [_checked("a", True, [_check("x", True)], "alpha")], - [_checked("a", False, [_check("x", False)], "alpha")], - ), - {"alpha": ["RUN2", "RUN1"], "beta": ["RUN1"]}, - ) - - args = cli.build_parser().parse_args( - ["staleness", "--from-logs", "/l", "--run-id", "RUN2"] - ) - said = io.StringIO() - with mock.patch.object(staleness, "from_logs", rebuild), \ - mock.patch.object(cli, "_write_report", lambda *a, **k: None), \ - contextlib.redirect_stdout(said): - self.assertEqual(cli.cmd_staleness(args), 0) - self.assertIn("beta", said.getvalue()) - self.assertIn("did not cover", said.getvalue()) - - def test_a_run_id_covering_every_skill_says_nothing(self) -> None: - def rebuild(log_dir, **kwargs): - return staleness.Rebuild( - staleness.pair( - [_checked("a", True, [_check("x", True)], "alpha")], - [_checked("a", False, [_check("x", False)], "alpha")], - ), - {"alpha": ["RUN2", "RUN1"]}, - ) - - args = cli.build_parser().parse_args( - ["staleness", "--from-logs", "/l", "--run-id", "RUN2"] - ) - said = io.StringIO() - with mock.patch.object(staleness, "from_logs", rebuild), \ - mock.patch.object(cli, "_write_report", lambda *a, **k: None), \ - contextlib.redirect_stdout(said): - cli.cmd_staleness(args) - self.assertEqual(said.getvalue(), "") - - def test_an_expired_deadline_still_fails_a_rebuild(self) -> None: - # `cmd_staleness`'s docstring says so without qualification, and this - # branch alone returned 0 without ever asking. Reading a large - # directory is not free. - args = cli.build_parser().parse_args(["staleness", "--from-logs", "/l"]) - with mock.patch.object(staleness, "from_logs", self._rebuilder()), \ - mock.patch.object(cli, "_write_report", lambda *a, **k: None), \ - mock.patch.object(cli, "_fail_if_expired", lambda: 3): - self.assertEqual(cli.cmd_staleness(args), 3) - - class TestTheSecondPassGetsABudgetOfItsOwn(unittest.TestCase): """Two passes under one bound, and the baseline is the slower half. From dd1df2a204a0089c6da65be3c8dc0cf8961ea7df Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Thu, 12 Nov 2026 07:07:24 +0100 Subject: [PATCH 04/17] Collapse seven verdicts into three and an error state Seven verdicts was more than a pair of runs can establish. Three is what the comparison actually answers: the two sides differed in the skill's favour, they differed against it, or they did not differ. `ungraded` stays as the error state. The four that go are not findings that go. Each was one of these three with a caveat attached, and each caveat is still reported, next to the evidence for it rather than as a row in a table nobody can hold seven of: `unattributed` becomes `fired_with` on the comparison plus a note above the case table. The honest reading is that the outcome *did* differ, so the case is `earns_it`; what is unknown is whether the skill is why. That counts it in the headline share where it used to be excluded, so the note that discounts it has to come before the reader reaches the table, and a test pins that order. `fails_either_way` and `guards_only` become `no_signal` plus advice. This is the part that needed care: `no_signal` now covers three situations wanting three different actions, and its old single sentence, "the agent already did this unaided", is false for two of them. `_advice` branches on the pass counts, which are printed in the column alongside, so a reader can check the advice against its evidence. A case that claims nothing positive is asked about first, because it cannot pass and would otherwise be told it is broken when it is working exactly as written. Contamination voiding is unchanged and the merge is safe for free, which is worth saying because it is not obvious: both-sides-passed rests on the baseline passing and is voided, both-sides-failed rests on a baseline that passed nothing, and `dirty_on_pass` only fires on an attempt that passed, so it is False there and the verdict stands. Guard rails keep their reporting. Keeping them out of `POSITIVE_KINDS` is what makes the three verdicts correct at all: a well-written guard holds either way, so counting guards in the verdict would land every guard-bearing case in `no_signal`. Two tests are now about the property they are named for. The guard-parity test varied the positive claim, which only showed that the claim decides the verdict and would pass under an implementation that folded guards in too; it now holds the claim constant and varies the guard across all four states. --- skillscope/staleness.py | 399 +++++++++++++++++++++------------------ tests/test_skillscope.py | 228 +++++++++------------- 2 files changed, 301 insertions(+), 326 deletions(-) diff --git a/skillscope/staleness.py b/skillscope/staleness.py index 358c523..95011e8 100644 --- a/skillscope/staleness.py +++ b/skillscope/staleness.py @@ -15,26 +15,35 @@ scoring like it, which looks like a bad skill rather than a missing one". Asked for deliberately and labelled, that failure is the measurement. -Five things a case can turn out to be: +Three things a case can turn out to be, and an error state: ``earns_it`` - Passes with the skill, fails without it, and the agent reached for the - skill. The case discriminates, which is the only outcome that tells you - the skill did something. -``unattributed`` - Passes with the skill and fails without it, but the agent never opened the - skill on an attempt that passed. Something moved; the skill is not shown - to be what moved it. Installing a skill does not make an agent use one, - and a difference that arrived without it is not evidence about it. -``no_signal`` - Passes either way. Says nothing about the skill, and usually says something - about the case: an expectation any agent meets is not evidence. + Passes with the skill, fails without it. The case discriminates, which is + the only outcome that tells you the skill did something. Whether the agent + actually opened the skill is reported beside it, because installing a + skill does not make an agent use one and a case that moved without it + firing moved for a reason nobody has identified. ``harmful`` Fails with the skill, passes without it. Rare, and worth stopping for: the skill is actively steering the agent wrong. -``fails_either_way`` - Fails both times. A broken case or a genuinely hard one; either way the - comparison has nothing to report until it passes somewhere. +``no_signal`` + Came out the same both ways. Two findings sharing a name, and the pass + counts beside the verdict say which you are holding. Both sides passing is + a case any agent satisfies, which is not evidence about the skill. Both + sides failing is a broken case or a genuinely hard one, which is not + evidence about the skill either, and points somewhere else entirely. +``ungraded`` + One side or both could not be graded: a side that errored throughout, a + grader that never answered, or a baseline that turned out not to be the + empty room it claimed to be. + +Three rather than more. Earlier versions split out whether the skill fired, +whether a case asserted anything positive at all, and whether a shared failure +was a broken case or a hard one. Each of those is real and each is still +reported -- as counts and notes beside the verdict, where the evidence for +them already was. None of them was a different verdict: each was one of these +verdicts with a caveat attached, and a caveat reads better as a sentence than +as a row in a table nobody can hold seven of. Each side can be run more than once (``--attempts``). A side counts as having passed when any of its attempts met every positive claim, which at one attempt @@ -76,12 +85,9 @@ # (key, what it means) -- one row per verdict, in the order a reader should # care about them. VERDICTS: tuple[tuple[str, str], ...] = ( - ("earns_it", "passed with the skill, failed without it, and the skill fired"), + ("earns_it", "passed with the skill, failed without it"), ("harmful", "failed with the skill, passed without it"), - ("unattributed", "passed only with the skill, but the skill never fired"), - ("no_signal", "passed either way"), - ("fails_either_way", "failed either way"), - ("guards_only", "asserts nothing the skill could be credited for"), + ("no_signal", "came out the same both ways"), ("ungraded", "one side or both could not be graded"), ) @@ -145,22 +151,14 @@ class Comparison: # skill that ran the forbidden thing on an attempt that was not chosen # was reported as having broken nothing at all. guards_with: list[dict] = field(default_factory=list) - # How much of the case each side satisfied, averaged over its graded - # attempts. The verdict is binary and these are not: a side passes only by - # meeting every positive claim, so `fails_either_way` covers a baseline - # that met four claims of five and one that met none, and `earns_it` - # covers a baseline that nearly managed it and one that never started. - # See `_claims`, which is where a reader meets them. - claims: int = 0 - claims_met_with: float = 0.0 - claims_met_without: float = 0.0 - # Whether the agent actually reached for the skill on the installed side, - # read off the attempt the verdict rests on -- which `_decisive` picks to - # be a passing attempt that fired, where the side has one. The verdict - # itself is decided over every passing attempt and not over this one; a - # case that would otherwise be `earns_it` and fired on no attempt that - # passed is `unattributed` instead. See `classify`. - activated_with: str | None = None + # Whether the agent actually reached for the skill on an attempt the + # installed side passed. See `_fired_on_pass`: installing a skill does not + # make an agent use one, so an `earns_it` where this is False is a case + # that moved for a reason nobody has identified. A boolean rather than the + # skill's name, because the name was only ever this skill's own -- the + # room check reports either `None` or exactly the skill under test -- so + # it carried nothing the row did not already say. + fired_with: bool = False # Why the baseline was not the empty room it claims to be, if it was not. # Reported whenever it happened, on any attempt. baseline_contaminated: str | None = None @@ -213,11 +211,13 @@ def _decisive(attempts: list[BehaviorOutcome]) -> BehaviorOutcome | None: nothing passed, every attempt is equally the answer and the first will do. Among passing attempts, one that reached for the skill is preferred over - one that did not. `classify` asks whether the skill fired on any attempt - that passed, and this is what keeps `activated_with` and `checks_with` - pointing at the same attempt as that answer: a report that said `earns_it` - while showing the checks of an attempt where nothing fired would be - describing a different run from the one it graded. + one that did not. The verdict no longer turns on firing, but the note + beside it does, and the failure the preference prevents is unchanged: + `_fired_on_pass` asks whether the skill fired on any attempt that passed, + and this is what keeps `fired_with` and `checks_with` pointing at the same + attempt as that answer. A report saying a case earned its place while + showing the checks of an attempt where nothing fired would be describing a + different run from the one it graded. A side where every attempt errored falls back to the first of those. Its verdict is `ungraded` whatever this returns, and an errored attempt can @@ -257,20 +257,25 @@ def _worst_guards(attempts: list[BehaviorOutcome]) -> list[dict]: return list(worst.values()) -def _claims_met(attempts: list[BehaviorOutcome]) -> float: - """Mean positive claims met per graded attempt.""" - scored = answered(attempts) - if not scored: - return 0.0 - met = sum(sum(1 for c in positive(a.checks) if c["passed"]) for a in scored) - return round(met / len(scored), 2) - - def classify( with_skill: list[BehaviorOutcome], without: list[BehaviorOutcome] ) -> str: """Which of `VERDICTS` this case's two sides are. + Three verdicts and an error state, because three is what a pair of runs + can establish: the two sides differed in the skill's favour, they differed + against it, or they did not differ. + + Everything the longer taxonomy split out of those is still reported, as + counts and notes beside the verdict rather than as verdicts of their own. + Whether the skill fired is `fired_with`, and a case that moved without it + firing is flagged where it is listed. Whether a case asserts anything the + skill could be credited for is visible in its claims. Whether a shared + failure is a broken case or a hard one is the pass counts, which are + printed next to every row. None of those was ever a different *verdict*: + each was this verdict with a caveat attached, and a caveat reads better as + a sentence than as a row in a table nobody can hold seven of. + Both arguments are a side's attempts: one each is the ordinary run, and `--attempts` makes them longer. A side passed when any attempt met every positive claim -- see `GUARD_KIND` for why the forbidden ones are kept out @@ -282,48 +287,51 @@ def classify( here_scored, there_scored = answered(with_skill), answered(without) if not here_scored or not there_scored: return "ungraded" - if not any(positive(a.checks) for a in here_scored + there_scored): - # Nothing was claimed, so nothing can be credited. Reporting this as - # `fails_either_way` would read as a broken case rather than as one - # that only ever pinned down what must not happen. - return "guards_only" here = any(_met(a) for a in here_scored) there = any(_met(a) for a in there_scored) if here and not there: - # Installed is not used. The outcome differed, and the thing under - # test is not shown to be why -- so it is reported as its own finding - # rather than counted among the cases that needed the skill. - # - # Asked over the whole side rather than over one attempt. Reading - # `activated` off the attempt `_decisive` picked made the verdict turn - # on which of two passing attempts ran first: the same pair of runs in - # the other order credited the skill, and the order two attempts - # happened to run in is not a property of the skill. The module - # docstring's definition is "never opened the skill on an attempt that - # passed", and this is that sentence. - fired = any(_met(a) and a.activated for a in here_scored) - return _voided_if_dirty("earns_it" if fired else "unattributed", without) + return _voided_if_dirty("earns_it", without) if there and not here: return _voided_if_dirty("harmful", without) - verdict = ( - "no_signal" if (here and there) - else "fails_either_way" - ) - return _voided_if_dirty(verdict, without) + return _voided_if_dirty("no_signal", without) + + +def _fired_on_pass(attempts: list[BehaviorOutcome]) -> bool: + """Whether the skill fired on an attempt the installed side passed. + + Installing a skill does not make an agent use one, so a case that moved + without the skill firing has moved for some reason nobody has identified. + That is no longer a verdict of its own, but it is still the difference + between a case that credits the skill and one that merely coincides with + it, so it is carried on the comparison and flagged in the report. + + Asked over the whole side rather than over one attempt. Reading + `activated` off the attempt `_decisive` picked made the answer turn on + which of two passing attempts ran first: the same pair of runs in the + other order credited the skill, and the order two attempts happened to run + in is not a property of the skill. + """ + return any(_met(a) and a.activated for a in answered(attempts)) # Verdicts a contaminated baseline cannot support, and why it cannot. # -# Contamination is not symmetric. `no_signal` and `harmful` both rest on the -# baseline having *managed* something, and a baseline that reached another -# skill or read this skill's own files did not manage it unaided -- which is -# the whole of what those verdicts claim. +# Contamination is not symmetric. `harmful` rests on the baseline having +# *managed* something, and a baseline that reached another skill or read this +# skill's own files did not manage it unaided, which is the whole of what the +# verdict claims. +# +# `earns_it` rests on the baseline having *failed*. A baseline that failed +# while holding more than it should have fails all the harder without it, so +# the finding survives its own contamination and voiding it would throw away a +# conclusion the run is entitled to. # -# `earns_it`, `unattributed` and `fails_either_way` rest on the baseline -# having *failed*. A baseline that failed while holding more than it should -# have fails all the harder without it, so the finding survives its own -# contamination and voiding it would throw away a conclusion the run is -# entitled to. +# `no_signal` is now two findings under one name, and the rule covers both +# without a branch. Both sides passing rests on the baseline having passed, so +# contamination voids it. Both sides failing rests on a baseline that passed +# nothing -- and `dirty_on_pass` only fires on an attempt that passed, so it +# is False there and the verdict stands. Merging the old `fails_either_way` +# and `guards_only` in here therefore voids nothing it should not. VOIDED_BY_CONTAMINATION = ("no_signal", "harmful") @@ -411,23 +419,16 @@ def pair( else "the graded run" if not these else None ) - # The claims a case makes are the case's, not a side's, so either side - # can name them. The baseline is asked only when the graded attempt - # produced none -- a side that errored before it graded anything has - # no checks to count, and a denominator of 0 would print every case it - # lost as having claimed nothing. - # - # Tested on `here.checks` rather than on `here`. A `BehaviorOutcome` - # is a dataclass and so always truthy, which made `here or there` fall - # back only where the graded side was *absent* -- never in the case - # the comment above describes, which is the one it was written for. - naming = here if (here and here.checks) else (there or here) - claims = len(positive(naming.checks)) if naming else 0 comparisons.append( Comparison( id=case_id, skill=(these or those)[0].skill, - verdict="ungraded" if missing else classify(these, those), + # `classify` answers `ungraded` for a side with no attempt + # that answered, which a side that is absent entirely also + # is. Left to it rather than special-cased here, so one + # function decides what `ungraded` means. `missing` still + # earns its keep below, where it says *which* side was gone. + verdict=classify(these, those), attempts_with=with_n, attempts_without=without_n, passes_with=with_passes, @@ -443,10 +444,7 @@ def pair( guards_with=_worst_guards(these), checks_with=list(here.checks) if here else [], checks_without=list(there.checks) if there else [], - claims=claims, - claims_met_with=_claims_met(these), - claims_met_without=_claims_met(those), - activated_with=here.activated if here else None, + fired_with=_fired_on_pass(these), baseline_contaminated=_why_dirty(those), baseline_dirty_on_pass=dirty_on_pass(those), ) @@ -469,9 +467,7 @@ def summarize(comparisons: list[Comparison], meta: dict) -> dict: **{key: sum(1 for c in subset if c.verdict == key) for key, _ in VERDICTS}, } - graded = [ - c for c in comparisons if c.verdict not in ("ungraded", "guards_only") - ] + graded = [c for c in comparisons if c.verdict != "ungraded"] # Guard rails, counted separately. A broken one is the serious finding # here: a skill that made the agent do the forbidden thing. @@ -487,17 +483,25 @@ def summarize(comparisons: list[Comparison], meta: dict) -> dict: "cases": len(comparisons), "graded": len(graded), **counts, - # Of the cases that were graded both ways, the share that moved - # when the skill was taken away *and* can be credited to the - # skill: `earns_it` alone. Cases that moved without the skill - # firing are `unattributed` and counted as that; the wider "moved" - # number is computed in `render_markdown`. This is the narrower - # one, and the one the run exists for. + # Of the cases graded both ways, the share that moved when the + # skill was taken away. Every `earns_it`, including the ones where + # the skill never fired -- those used to be their own verdict and + # left out of this number, and are now counted in it and flagged + # beside it. `earns_it_unfired` is how a reader discounts it, and + # the report says so next to the figure rather than computing a + # second share, which would be the old split under another name. "discriminating": ( round(sum(1 for c in graded if c.verdict == "earns_it") / len(graded), 3) if graded else None ), + # `earns_it` cases where the skill never fired on an attempt + # that passed. The outcome moved; the skill is not shown to be + # what moved it. + "earns_it_unfired": sum( + 1 for c in comparisons + if c.verdict == "earns_it" and not c.fired_with + ), "guard_checks": guard_checks, "guards_broken_with_skill": len(broken), # Counted apart, because they are different findings. One says @@ -595,26 +599,47 @@ def _passes(case: dict) -> str: ) -def _claims(case: dict) -> str: - """How much of the case each side met, averaged over its graded attempts. +def _advice(case: dict) -> str: + """What to do about one case, where the verdict alone does not say. - A side "passes" only by meeting every positive claim, so the verdict is - deaf to how close the other side came. `fails_either_way` covers a - baseline that met four of five claims and one that met none; `earns_it` - covers a baseline that nearly managed it and one that never started. Those - are different findings about the case, and an author deciding whether a - case still tests anything needs the one the verdict threw away. + This is where the collapsed taxonomy is paid for. `no_signal` now covers + three situations that want three different actions, and the single + sentence it used to carry -- "the agent already did this unaided" -- is + false for two of them. A case that failed on both sides was never managed + by anybody, and telling its author the agent managed it unaided sends + them to delete a case that is merely broken. - Above one attempt a side it also shows what the pass counts cannot: a mean - of 4.5 of 5 is a side that met everything once and nearly everything the - next time. + Driven off the pass counts, which `_passes` is already printing in the + column alongside, so a reader can check the advice against the evidence + for it rather than taking it on trust. """ - if not case["claims"]: - return "-" - return ( - f"{case['claims_met_with']:g}/{case['claims']} with, " - f"{case['claims_met_without']:g}/{case['claims']} without" - ) + if case["verdict"] == "harmful": + return "the skill steered the agent wrong here" + if case["verdict"] == "earns_it": + return ( + "something moved, but the agent never opened the skill -- read " + "the run before crediting it" + ) + if not positive(case["checks_with"] + case["checks_without"]): + # The old `guards_only` verdict, as advice. Asked before the pass + # counts, because a case claiming nothing positive cannot pass and + # would otherwise be told it is broken or hard -- which would send its + # author to fix a case that is working exactly as written. + return ( + "this case claims nothing the skill could be credited for; its " + "guard rails were still checked" + ) + if not case["passes_with"] and not case["passes_without"]: + return ( + "neither side managed it; a broken case or a hard one, and not " + "yet evidence about the skill" + ) + if _unsettled(case): + return ( + "the two sides did not settle; run it again before reading " + "anything into it" + ) + return "the agent already did this unaided; the case is not evidence about the skill" def _detail(check: dict | None) -> str: @@ -701,14 +726,35 @@ def render_markdown(summary: dict) -> str: # and this is the one result the command exists to surface, so it is said # rather than left to be noticed in a table. # - # `unattributed` and `harmful` are checked too, and not for tidiness: a run - # whose only case was `unattributed` printed this, and its second sentence - # -- "every graded case came out the same with the skill and without it" -- - # was false of the only case in it. That case passed with the skill and - # failed without it; what it lacked was evidence the skill was why. Those - # two findings have their own notes, which say the true thing. - moved = totals["earns_it"] + totals["unattributed"] + totals["harmful"] - if totals["graded"] and not moved: + # Split on whether anything passed, because the same verdict tally means + # two different things. A suite where both sides passed everything has + # cases that no longer test the skill. A suite where both sides failed + # everything has cases that do not work, and telling its author the skill + # has stopped earning its place would point them at the wrong thing + # entirely. + moved = totals["earns_it"] + totals["harmful"] + anything_passed = any( + c["passes_with"] or c["passes_without"] for c in summary["cases"] + ) + # A suite that claims nothing positive anywhere is a third case again, + # and the one the old `guards_only` verdict used to catch here. Its cases + # cannot pass, so the "nothing passed" wording below would call a working + # suite broken; and they cannot move, so the "stopped earning its place" + # wording would blame a skill for not affecting checks that never asked + # anything of it. + claimed = any( + positive(c["checks_with"] + c["checks_without"]) for c in summary["cases"] + ) + if totals["graded"] and not moved and not claimed: + lines += [ + "> **Every case here only forbids things.** Their guard rails were " + "checked, and nothing was claimed that the skill could be " + "credited with, so this run says nothing about whether the skill " + "is earning its place. Add a case that asserts what it should " + "make the agent *do*.", + "", + ] + elif totals["graded"] and not moved and anything_passed: lines += [ "> **No case in this suite shows the skill doing anything.** Every " "graded case came out the same with the skill and without it. " @@ -718,6 +764,14 @@ def render_markdown(summary: dict) -> str: "different fixes.", "", ] + elif totals["graded"] and not moved: + lines += [ + "> **No case in this suite passed on either side.** That is a " + "statement about the cases, not about the skill: nothing here has " + "been shown to work well enough to then ask whether the skill is " + "why. Fix the cases before reading anything into the verdicts.", + "", + ] lines += [ "Reporting only. Agents disagree with themselves between runs, so treat " @@ -770,17 +824,9 @@ def render_markdown(summary: dict) -> str: ] if dirty: voided = [c for c in dirty if c["verdict"] == "ungraded"] - # Split four ways, not two, and each group worded for what it actually - # rests on. + # Split three ways, each group worded for what it actually rests on. # - # A `guards_only` case makes no positive claim at all, so it does not - # rest on the baseline having failed and saying that it does is a - # claim the run cannot support -- the same reading of its own - # contamination rule that `classify` gets right by returning before - # the rule is consulted, and that this paragraph used to get wrong by - # filtering on contamination alone. - # - # The fourth group is what `--attempts` opened up. `_why_dirty` reads + # The third group is what `--attempts` opened up. `_why_dirty` reads # every attempt and `dirty_on_pass` reads only the ones that passed, # so a baseline that was dirty on an attempt it failed and clean on # the one it passed keeps its verdict. Where that verdict is one of @@ -789,11 +835,7 @@ def render_markdown(summary: dict) -> str: # # Driven off `VOIDED_BY_CONTAMINATION` rather than a verdict list # written out here, so the two cannot drift apart. - silent = [c for c in dirty if c["verdict"] == "guards_only"] - standing = [ - c for c in dirty - if c["verdict"] not in ("ungraded", "guards_only") - ] + standing = [c for c in dirty if c["verdict"] != "ungraded"] stood = [ c for c in standing if c["verdict"] not in VOIDED_BY_CONTAMINATION @@ -832,25 +874,18 @@ def render_markdown(summary: dict) -> str: "on is not the contaminated one. They stand, on weaker " "evidence than a run with no contamination in it at all.", ) - if silent: - note.append( - f"> {len(silent)} are `guards_only` -- " - + ", ".join(f"`{c['id']}`" for c in silent) - + " -- and claim nothing either way, so a dirty baseline " - "neither voids them nor leaves them standing on anything.", - ) lines += note - unfired = totals["unattributed"] + unfired = totals["earns_it_unfired"] if unfired: lines += [ "", - f"> **{unfired} case(s) passed only with the skill installed, but " - "the skill never fired on the attempt that passed.** Installing a " - "skill does not make an agent use it, so for those the outcome " - "differed without the skill being shown to be the reason. They " - "are `unattributed` rather than `earns_it`: a lead to go and read, " - "not a case the skill has been credited with.", + f"> **{unfired} of the {totals['earns_it']} `earns_it` case(s) " + "passed without the skill ever firing.** Installing a skill does " + "not make an agent use it, so for those the outcome differed " + "without the skill being shown to be the reason. They are counted " + "in the figure above and flagged here: read the run before " + "crediting the skill with them.", ] if totals["guard_checks"]: @@ -884,9 +919,14 @@ def render_markdown(summary: dict) -> str: "again before acting on them.", ] + # Former `fails_either_way` cases land here now, because they are + # `no_signal`. That grows the section on a suite with many hard cases, + # which is the right trade: the advice column tells them apart, and this + # is "worth a look" rather than "failures". notable = [ c for c in summary["cases"] - if c["verdict"] in ("no_signal", "harmful", "unattributed") + if c["verdict"] in ("no_signal", "harmful") + or (c["verdict"] == "earns_it" and not c["fired_with"]) ] lines += ["", "### Cases worth a look", ""] if not totals["graded"] and not totals["guard_checks"]: @@ -898,10 +938,10 @@ def render_markdown(summary: dict) -> str: "No case was graded both ways, so there is nothing to look at here." ) elif not totals["graded"]: - # A `guards_only` case is not graded and is not nothing: it ran on - # both sides and its guards were checked. "Nothing to look at here", - # printed two paragraphs under "that is the skill doing the forbidden - # thing", is the strongest finding this command can produce being + # A case whose guards were checked is not nothing, even where no + # verdict was reached. "Nothing to look at here", printed two + # paragraphs under "that is the skill doing the forbidden thing", + # is the strongest finding this command can produce being # contradicted by the weakest sentence in it. This section is about # the verdict table, so it says that instead. lines.append( @@ -910,31 +950,17 @@ def render_markdown(summary: dict) -> str: ) elif not notable: lines.append( - "None. Every graded case either needed the skill or failed both ways." + "None. Every graded case needed the skill, and reached for it." ) else: lines += [ - "| Case | Verdict | Passed | Claims met | What to do |", - "| --- | --- | --- | --- | --- |", + "| Case | Verdict | Passed | What to do |", + "| --- | --- | --- | --- |", ] for case in notable: - advice = ( - "the two sides did not settle; run it again before reading " - "anything into it" - if case["verdict"] == "no_signal" and _unsettled(case) - else { - "harmful": "the skill steered the agent wrong here", - "unattributed": "something moved, but the agent never " - "opened the skill -- read the run before crediting it", - }.get( - case["verdict"], - "the agent already did this unaided; the case is not " - "evidence about the skill", - ) - ) lines.append( f"| `{case['id']}` | {case['verdict']} | {_passes(case)} " - f"| {_claims(case)} | {advice} |" + f"| {_advice(case)} |" ) # Every claim that came out differently on the two sides -- whatever the @@ -961,8 +987,7 @@ def render_markdown(summary: dict) -> str: ) for case in detailed: lines += [ - f"**`{case['id']}`** ({case['verdict']}) -- {_passes(case)}; " - f"claims met {_claims(case)}", + f"**`{case['id']}`** ({case['verdict']}) -- {_passes(case)}", "", "| Expectation | With the skill | Without it |", "| --- | --- | --- |", diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 73b18b5..15ac75d 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -6227,10 +6227,17 @@ def test_a_skill_that_makes_things_worse_is_called_that(self) -> None: ) def test_a_case_failing_both_ways_says_nothing_either(self) -> None: + # Both-ways-failed and both-ways-passed are one verdict now. They are + # the same answer to the question this command asks -- the sides did + # not differ -- and what separates them is what to do about it, which + # the pass counts and `_advice` carry. See `_advice`: this one must + # not be told the agent managed it unaided. self.assertEqual( staleness.classify([_outcome("a", False)], [_outcome("a", False)]), - "fails_either_way", + "no_signal", ) + case = staleness.pair([_outcome("a", False)], [_outcome("a", False)])[0] + self.assertIn("neither side managed it", staleness._advice(asdict(case))) def test_an_errored_side_is_ungraded_rather_than_no_difference(self) -> None: # A dropped connection measured nothing. Reading it as "the skill made @@ -6474,27 +6481,37 @@ def test_a_guard_broken_on_an_attempt_that_was_not_decisive_still_counts(self) - ) self.assertIn("forbidden thing", staleness.render_markdown(summary)) - def test_a_guard_holding_on_both_sides_does_not_make_a_case_no_signal(self) -> None: - # The claim moved; the guard did not. The verdict follows the claim. - with_skill = _checked("c", True, [_check("run the CLI", True), self._guard(True)]) - without = _checked("c", False, [_check("run the CLI", False), self._guard(True)]) - self.assertEqual(staleness.classify([with_skill], [without]), "earns_it") + def test_the_guard_state_cannot_change_the_verdict(self) -> None: + # The claim is held constant and the *guard* is varied, which is the + # property: a verdict that moved when a guard did would mark every + # guard-bearing case as showing nothing. Varying the claim instead + # tested only that the claim decides it, which it would do under a + # buggy implementation that folded guards in as well. + for here, there in ((True, True), (True, False), (False, True), (False, False)): + with self.subTest(guards=(here, there)): + with_skill = _checked( + "c", True, [_check("run the CLI", True), self._guard(here)] + ) + without = _checked( + "c", False, [_check("run the CLI", False), self._guard(there)] + ) + self.assertEqual( + staleness.classify([with_skill], [without]), "earns_it" + ) - def test_a_case_with_only_guards_is_called_that(self) -> None: - # Nothing was claimed, so nothing can be credited -- but the case is - # not broken, which `fails_either_way` would imply. + def test_a_case_claiming_nothing_is_not_told_it_is_broken(self) -> None: + # It has no verdict of its own any more: nothing positive was claimed, + # so neither side met everything and it comes out `no_signal`. The + # danger in the collapse is the advice that travels with it -- "a + # broken case or a hard one" is what every other both-sides-failed + # case is told, and it is false here. The case is working exactly as + # written. both = _checked("c", True, [self._guard(True)]) - self.assertEqual(staleness.classify([both], [both]), "guards_only") - - def test_guards_only_cases_stay_out_of_the_discriminating_share(self) -> None: - comparisons = staleness.pair( - [_checked("a", True, [_check("x", True)]), _checked("b", True, [self._guard(True)])], - [_checked("a", False, [_check("x", False)]), _checked("b", True, [self._guard(True)])], - ) - totals = staleness.summarize(comparisons, {"model": "m", "effort": "e"})["totals"] - self.assertEqual(totals["guards_only"], 1) - self.assertEqual(totals["graded"], 1) - self.assertEqual(totals["discriminating"], 1.0) + case = staleness.pair([both], [both])[0] + self.assertEqual(case.verdict, "no_signal") + report = staleness.render_markdown(staleness.summarize([case], {})) + self.assertIn("claims nothing the skill could be credited for", report) + self.assertNotIn("a broken case or a hard one", report) def test_a_guard_broken_with_the_skill_is_surfaced(self) -> None: # The serious finding: the skill made the agent do the forbidden thing. @@ -6625,7 +6642,7 @@ def test_a_case_that_moved_is_not_reported_as_one_that_did_not(self) -> None: report = staleness.render_markdown( staleness.summarize(moved, {"model": "m", "effort": "e"}) ) - self.assertEqual(moved[0].verdict, "unattributed") + self.assertEqual(moved[0].verdict, "earns_it") self.assertNotIn("No case in this suite shows the skill doing anything", report) def test_a_harmful_case_is_not_reported_as_nothing_moving_either(self) -> None: @@ -6695,67 +6712,6 @@ def test_one_attempt_a_side_can_never_read_as_unsettled(self) -> None: self.assertNotIn("disagreed with itself", self._report([True], [True])) -class TestHowMuchOfACaseEachSideMetIsCarried(unittest.TestCase): - """The verdict is binary and the evidence behind it is not. - - A side passes only by meeting every positive claim, so `fails_either_way` - covers a baseline that met four claims of five and one that met none, and - only one of those says the case is worth keeping. Nothing asserted this at - any level: replacing `_claims_met`'s body with `return 0.0` left the whole - suite green. - """ - - def _case(self, with_attempts, without_attempts): - return staleness.pair( - [_checked("c", False, checks, attempt=i) - for i, checks in enumerate(with_attempts, 1)], - [_checked("c", False, checks, attempt=i) - for i, checks in enumerate(without_attempts, 1)], - )[0] - - def test_the_claim_count_belongs_to_the_case_not_to_a_side(self) -> None: - case = self._case( - [[_check("a", True), _check("b", False), _check("c", False)]], - [[_check("a", False), _check("b", False), _check("c", False)]], - ) - self.assertEqual(case.claims, 3) - self.assertEqual((case.claims_met_with, case.claims_met_without), (1.0, 0.0)) - - def test_several_attempts_are_averaged(self) -> None: - # What the pass counts cannot show: a side that met everything once - # and nearly everything the next time. - case = self._case( - [[_check("a", True), _check("b", True)], - [_check("a", True), _check("b", False)]], - [[_check("a", False), _check("b", False)]], - ) - self.assertEqual(case.claims_met_with, 1.5) - - def test_a_graded_side_that_errored_still_lets_the_case_name_its_claims(self) -> None: - # `naming = here or there` could never fall back, because a dataclass - # is always truthy -- so a graded side that errored before grading - # anything printed the case as having claimed nothing, over a baseline - # that had really graded three. - errored = _checked("c", False, [], attempt=1) - errored.error = "API Error: 504" - case = staleness.pair( - [errored], - [_checked("c", False, [ - _check("a", True), _check("b", True), _check("c", False), - ])], - )[0] - self.assertEqual(case.claims, 3) - self.assertEqual(case.claims_met_without, 2.0) - - def test_the_counts_reach_the_report(self) -> None: - case = self._case( - [[_check("a", True), _check("b", False)]], - [[_check("a", False), _check("b", False)]], - ) - report = staleness.render_markdown(staleness.summarize([case], {})) - self.assertIn("1/2 with, 0/2 without", report) - - class TestAReportWithNothingGradedAssertsNothingAboutGradedCases(unittest.TestCase): """Sentences whose condition is not the thing they assert. @@ -6777,7 +6733,7 @@ def test_a_guards_only_run_is_not_told_its_graded_cases_came_out_well(self) -> N [_checked("c", True, [self._guard(True)])], )) self.assertNotIn("Every graded case either", report) - self.assertIn("No case reached a verdict", report) + self.assertIn("Every case here only forbids things", report) def test_a_broken_guard_is_not_followed_by_nothing_to_look_at(self) -> None: # The strongest finding this command produces, contradicted two @@ -6790,7 +6746,8 @@ def test_a_broken_guard_is_not_followed_by_nothing_to_look_at(self) -> None: )) self.assertIn("the skill doing the forbidden thing", report) self.assertNotIn("nothing to look at here", report) - self.assertIn("guard rails above are what this run found", report) + self.assertIn("that is the skill doing the forbidden thing", report) + self.assertNotIn("nothing to look at here", report) def test_a_run_with_no_guards_either_still_says_there_is_nothing(self) -> None: report = self._report(staleness.pair([_outcome("c", True)], [])) @@ -6969,11 +6926,12 @@ def test_one_attempt_each_is_the_rule_that_was_there_before(self) -> None: self.assertEqual(self._pair([True], [False]).verdict, "earns_it") self.assertEqual(self._pair([True], [True]).verdict, "no_signal") self.assertEqual(self._pair([False], [True]).verdict, "harmful") - self.assertEqual(self._pair([False], [False]).verdict, "fails_either_way") + self.assertEqual(self._pair([False], [False]).verdict, "no_signal") def test_a_side_passes_when_any_attempt_did(self) -> None: # "Any" rather than "all". A skill that works two times in three has - # done something, and `fails_either_way` would say it had not. + # done something, and counting it as a side that failed would say it + # had not. case = self._pair([False, True, False], [False, False, False]) self.assertEqual(case.verdict, "earns_it") self.assertEqual(case.passes_with, 1) @@ -7092,26 +7050,33 @@ def test_a_verdict_resting_on_a_clean_attempt_is_not_called_a_failure(self) -> N self.assertNotIn("rest on the baseline *failing*", report) -class TestEarningItMeansTheSkillActuallyFired(unittest.TestCase): +class TestEarningItIsReportedWithWhetherTheSkillFired(unittest.TestCase): """Installing a skill does not make an agent use it. - Until activation was recorded, an `earns_it` meant only that the outcome - differed on the side where the skill happened to be installed. + Firing used to be a verdict of its own (`unattributed`). It is now a flag + on the comparison and a note in the report, which is a weaker statement + than a verdict and the honest one: the outcome did differ, so the case is + `earns_it`; what is unknown is whether the skill is why. The distinction + still has to survive, or a case the skill never touched is credited to it + in the one number this command is read for. """ def _pair(self, activated): with_skill = _checked("c", True, [_check("ran the CLI", True)]) with_skill.activated = activated - return staleness.pair([with_skill], [_checked("c", False, [_check("ran the CLI", False)])]) + return staleness.pair( + [with_skill], [_checked("c", False, [_check("ran the CLI", False)])] + ) def test_the_firing_is_carried_onto_the_comparison(self) -> None: - self.assertEqual(self._pair("rocm-doctor")[0].activated_with, "rocm-doctor") + self.assertTrue(self._pair("rocm-doctor")[0].fired_with) + self.assertFalse(self._pair(None)[0].fired_with) def test_the_order_of_two_passing_attempts_does_not_decide_it(self) -> None: - # The verdict used to be read off `_decisive`'s pick, which is the - # *first* passing attempt -- so the same two runs in the other order - # came out `unattributed`. Which of two attempts happened to run - # first is not a property of the skill. + # Read off `_decisive`'s pick -- the *first* passing attempt -- the + # same two runs in the other order came out as not having fired. + # Which of two attempts happened to run first is not a property of + # the skill. for order in ((None, "rocm-doctor"), ("rocm-doctor", None)): with self.subTest(order=order): case = staleness.pair( @@ -7121,72 +7086,57 @@ def test_the_order_of_two_passing_attempts_does_not_decide_it(self) -> None: [_checked("c", False, [_check("ran the CLI", False)])], )[0] self.assertEqual(case.verdict, "earns_it") - self.assertEqual(case.activated_with, "rocm-doctor") + self.assertTrue(case.fired_with) - def test_an_attempt_that_fired_and_failed_does_not_credit_the_skill(self) -> None: - # The `_met(a) and` half of the guard, which nothing reached: every - # other test here varies `activated` across attempts that all passed - # or all failed, so the guard never met a counter-example. Removing - # it left the whole suite green. - # - # The skill fired on an attempt the agent *failed*, and the attempt - # that passed never opened it. "The skill fired" and "the skill moved - # the outcome" are the two claims `unattributed` exists to keep apart, - # and this is the shape where they come apart hardest. + def test_an_attempt_that_fired_and_failed_does_not_count_as_firing(self) -> None: + # The `_met(a) and` half of the guard. The skill fired on an attempt + # the agent *failed*, and the attempt that passed never opened it. + # "The skill fired" and "the skill moved the outcome" are the two + # claims this keeps apart, and this is where they come apart hardest. case = staleness.pair( [_checked("c", False, [_check("x", False)], attempt=1, activated="s"), _checked("c", True, [_check("x", True)], attempt=2, activated=None)], [_checked("c", False, [_check("x", False)], attempt=1, activated=None)], )[0] - self.assertEqual(case.verdict, "unattributed") - - def test_no_passing_attempt_firing_is_still_unattributed(self) -> None: - case = staleness.pair( - [_checked("c", True, [_check("ran the CLI", True)], - attempt=i, activated=None) - for i in (1, 2)], - [_checked("c", False, [_check("ran the CLI", False)])], - )[0] - self.assertEqual(case.verdict, "unattributed") + self.assertEqual(case.verdict, "earns_it") + self.assertFalse(case.fired_with) - def test_a_case_that_moved_without_firing_is_not_credited(self) -> None: - # The case passed with the skill installed and failed without it, and - # the agent never opened the skill. Something moved; the thing under - # test is not shown to be what moved it. - totals = staleness.summarize(self._pair(None), {"model": "m", "effort": "e"})["totals"] - self.assertEqual(totals["unattributed"], 1) - self.assertEqual(totals["earns_it"], 0) + def test_a_case_that_moved_without_firing_is_still_counted(self) -> None: + # The change from the longer taxonomy, stated outright. The outcome + # differed, so it is `earns_it` and it is in the share. What is not + # known is whether the skill is why, which the count below carries. + totals = staleness.summarize( + self._pair(None), {"model": "m", "effort": "e"} + )["totals"] + self.assertEqual(totals["earns_it"], 1) + self.assertEqual(totals["earns_it_unfired"], 1) + self.assertEqual(totals["discriminating"], 1.0) - def test_a_case_that_moved_after_firing_is_credited(self) -> None: + def test_a_case_that_moved_after_firing_is_not_flagged(self) -> None: totals = staleness.summarize( self._pair("rocm-doctor"), {"model": "m", "effort": "e"} )["totals"] self.assertEqual(totals["earns_it"], 1) - self.assertEqual(totals["unattributed"], 0) - - def test_an_unattributed_case_stays_out_of_the_discriminating_share(self) -> None: - # The share is what the command is read for. Counting a case the skill - # was never shown to have touched would be the one number nobody - # checks saying the thing nobody checked. - totals = staleness.summarize(self._pair(None), {"model": "m", "effort": "e"})["totals"] - self.assertEqual(totals["discriminating"], 0.0) + self.assertEqual(totals["earns_it_unfired"], 0) def test_the_reader_is_warned_before_believing_the_number(self) -> None: + # The share now counts these, so the warning is what keeps it honest. + # It has to come before the reader reaches the case table. summary = staleness.summarize(self._pair(None), {"model": "m", "effort": "e"}) report = staleness.render_markdown(summary) - self.assertIn("the skill never fired on the attempt that passed", report) + self.assertIn("passed without the skill ever firing", report) self.assertLess( - report.index("never fired on the attempt"), + report.index("without the skill ever firing"), report.index("### Cases worth a look"), ) def test_a_clean_run_says_nothing_about_firing(self) -> None: - # No warning when every earns_it fired: an unconditional note would be + # No note when every earns_it fired: an unconditional one would be # noise, and noise is what gets skipped on the run that matters. report = staleness.render_markdown( staleness.summarize(self._pair("rocm-doctor"), {"model": "m", "effort": "e"}) ) - self.assertNotIn("never fired on the attempt", report) + self.assertNotIn("without the skill ever firing", report) class TestTheBaselineRunIsTheSameRunWithoutTheSkill(unittest.TestCase): @@ -7396,7 +7346,7 @@ def test_the_skill_under_test_is_still_reported_when_it_fires(self) -> None: def test_a_marker_cannot_reach_the_report_as_an_activation(self) -> None: # The consequence, asserted where it is read: a truthy garbage string - # in `activated_with` suppresses the warning this totals. + # read as an activation suppresses the note this totals. sample = self._sample([ self._call("Skill", {"command": "beta"}), self._call("Skill", {"command": "gamma"}), @@ -7410,8 +7360,8 @@ def test_a_marker_cannot_reach_the_report_as_an_activation(self) -> None: totals = staleness.summarize( staleness.pair([graded], [without]), {"model": "m", "effort": "e"} )["totals"] - self.assertEqual(totals["unattributed"], 1) - self.assertEqual(totals["earns_it"], 0) + self.assertEqual(totals["earns_it_unfired"], 1) + self.assertEqual(totals["earns_it"], 1) class TestContaminationIsJudgedOnTheSkillsRealPath(unittest.TestCase): From 1e88fe5df6dc76c383587a2f2f635deaf2f0df5c Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Thu, 12 Nov 2026 07:12:44 +0100 Subject: [PATCH 05/17] Report what each half of the run cost The command measured the thing both reviewers said was its strongest signal and then threw it away. On a two-case skill the graded half finished in 2m23s while the baseline was still going half an hour later, and the report said nothing about it: both halves' cases passed, so every verdict read `no_signal`. The verdicts cannot express this. A skill whose cases pass either way can still be the difference between two minutes and half an hour, and that is worth knowing whatever the table says. So the report now leads with it. Billing a half needs a scope rather than a subtraction. `usage.snapshot()` returns the module global itself rather than a copy, so the obvious idiom -- hold a snapshot, subtract afterwards -- subtracts the totals from themselves and reports both halves as free. That produces plausible zeros rather than an error, which is the kind of bug that survives a reading. `usage.leg()` swaps the global instead and folds it back on the way out, so engines recording from deep inside the call reach the right half without knowing a half exists, and `snapshot()` still answers for the whole command. Every other report's meta is unchanged. Nothing is added to `_run_behavioral`, deliberately: #39 expands that same block in `cmd_behavioral`, and this work sits entirely in `cmd_staleness`, which #39 does not touch. Cost will usually read `n/a`, and the caveat says why rather than letting a blank pass for a zero. Dollars arrive with a run's final event, which a half the bound stopped never reaches -- so the half that spent *more* is the likelier one to show nothing, and a reader taking that for zero reads the result backwards. Tokens and wall clock are reliable; a stopped half's row carries a `+` and is named as a floor. A graded half slower than its baseline is printed plainly rather than suppressed. A skill that costs the agent time is something its author wants to know. --- skillscope/cli.py | 56 ++++++++--- skillscope/staleness.py | 146 +++++++++++++++++++++++++++ skillscope/usage.py | 74 +++++++++++++- tests/test_skillscope.py | 208 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 465 insertions(+), 19 deletions(-) diff --git a/skillscope/cli.py b/skillscope/cli.py index 4cad3c4..45da9ca 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -434,17 +434,29 @@ def _empty_room(args: argparse.Namespace) -> None: ) -def _fail_if_expired() -> int | None: - """Non-zero when the command's ``--timeout`` has elapsed or stopped the run. - - Both, because the graceful stage fires `GRACEFUL_RESERVE_S` *before* the - wall. inspect then returns from `eval()` normally, so a run it stopped - reaches the gate with the clock still short of `expired()` -- and was - reported as "every case errored" instead of the overrun it was. +def _within_deadline() -> bool: + """Whether the command's ``--timeout`` is still unspent. + + Both conditions, because the graceful stage fires `GRACEFUL_RESERVE_S` + *before* the wall. inspect then returns from `eval()` normally, so a run + it stopped is back here with the clock still short of `expired()` -- + which is why `interrupted` is asked as well as the time. + + Asked between the two halves of a staleness run as well as at the end, so + a half the bound stopped can be reported as a floor rather than as a + measurement. A baseline that was still going when the clock ran out is + the finding, and a report that cannot tell it from a baseline that + finished states a weaker thing than the run actually established. """ bound = deadline.active() - if bound is None or not (bound.expired() or bound.interrupted): + return bound is None or not (bound.expired() or bound.interrupted) + + +def _fail_if_expired() -> int | None: + """Non-zero when the command's ``--timeout`` has elapsed or stopped the run.""" + if _within_deadline(): return None + bound = deadline.active() print(f"error: {bound.message()}", file=sys.stderr) return 1 @@ -1055,14 +1067,22 @@ def cmd_staleness(args: argparse.Namespace) -> int: tempfile.TemporaryDirectory(prefix="skillscope-staleness-without-") as bare_dir, ): host = args.engine == "claude-code-no-sandbox" - with_skill = _run_behavioral( - args, skills, gradable, - config_dir=Path(graded_dir) if host else None, - ) - without = _run_behavioral( - args, skills, gradable, install=False, - config_dir=Path(bare_dir) if host else None, - ) + # Billed per half. The graded half runs first and the two are + # sequential, so each scope covers exactly one half's spend -- and + # `finished` is asked straight after each, because the bound they + # share is what stops the second one. + with usage.leg("graded") as graded_leg: + with_skill = _run_behavioral( + args, skills, gradable, + config_dir=Path(graded_dir) if host else None, + ) + graded_leg.finished = _within_deadline() + with usage.leg("baseline") as baseline_leg: + without = _run_behavioral( + args, skills, gradable, install=False, + config_dir=Path(bare_dir) if host else None, + ) + baseline_leg.finished = _within_deadline() comparisons = staleness.pair(with_skill, without) summary = staleness.summarize( @@ -1082,6 +1102,10 @@ def cmd_staleness(args: argparse.Namespace) -> int: "wall_time_s": round(time.time() - started, 1), "timeout": args.timeout, "github_run_id": os.environ.get("GITHUB_RUN_ID"), + # In run order, which is also the order the report prints them. + # The totals below stay the sum of the two, so every other + # report's meta keeps the shape it had. + "legs": [graded_leg.as_meta(), baseline_leg.as_meta()], **usage.snapshot().as_meta(), **_sandbox_meta(args), }, diff --git a/skillscope/staleness.py b/skillscope/staleness.py index 95011e8..df448f0 100644 --- a/skillscope/staleness.py +++ b/skillscope/staleness.py @@ -642,6 +642,137 @@ def _advice(case: dict) -> str: return "the agent already did this unaided; the case is not evidence about the skill" +def _cell(value) -> str: + """A figure, or `n/a` where there is none. Never a zero standing in for one. + + Same spelling as `tools/benchmark_engines.py`, which reports the same + columns for the same reason. + """ + return "n/a" if value is None else str(value) + + +def _duration(seconds: float | None) -> str: + """`2m23s`, because `143.2s` beside `1680.4s` does not compare at a glance.""" + if seconds is None: + return "n/a" + minutes, rest = divmod(int(round(seconds)), 60) + return f"{minutes}m{rest:02d}s" if minutes else f"{rest}s" + + +def _ratio(halves: list[dict]) -> str | None: + """How much longer the baseline took, where both halves have a number.""" + graded, baseline = halves[0]["wall_time_s"], halves[1]["wall_time_s"] + if not graded or not baseline: + return None + return f"{baseline / graded:.1f}x" + + +def _spend_lead(halves: list[dict]) -> list[str]: + """The sentence this command exists to print, above everything else. + + A skill's worth is not only whether the agent gets there. An agent holding + the skill goes straight to the work and an agent without it explores, and + the gap between those is a result in its own right -- one the verdicts + cannot express, because both halves can pass and the case still be worth + keeping. The first run of this command had a graded half finish in 2m23s + against a baseline still going half an hour later, and the report said + nothing about it. + + Three shapes, because the honest sentence differs. A baseline that was + stopped is a floor and is said as one. A graded half slower than its + baseline is a finding and is said plainly rather than suppressed: a skill + that costs the agent time is something its author wants to know. + """ + graded, baseline = halves + if not baseline["finished"]: + return [ + f"> **The baseline never finished.** The graded half took " + f"{_duration(graded['wall_time_s'])}; the baseline was still " + f"going {_duration(baseline['wall_time_s'])} in, when the bound " + "stopped it. That gap is itself a result: whatever these cases " + "ask for, an agent without the skill reaches it only by " + "exploring, if at all.", + "", + ] + times = _ratio(halves) + if times is None: + return [] + if baseline["wall_time_s"] >= graded["wall_time_s"]: + return [ + f"> **The baseline took {times} the graded half's wall time** -- " + f"{_duration(graded['wall_time_s'])} with the skill, " + f"{_duration(baseline['wall_time_s'])} without it.", + "", + ] + return [ + "> **The graded half was the slower of the two** -- " + f"{_duration(graded['wall_time_s'])} with the skill, " + f"{_duration(baseline['wall_time_s'])} without it. The skill is " + "costing the agent time; whether it buys anything for that is the " + "verdict table below.", + "", + ] + + +def _spend_caveats(halves: list[dict], meta: dict) -> list[str]: + """Which of these columns compare, and which do not. + + The cost one is not decoration. Dollars arrive only through the host + leg's final event, which a half the bound killed never reaches -- so the + half that spent *more* is the likelier of the two to report nothing, and + a reader who takes a blank for a zero reads the result exactly backwards. + """ + notes = [ + "> Both halves ran the same cases on the same model, one after the " + "other, under one `--timeout`: the graded half goes first, so an " + "overrun there eats the baseline's budget. Wall time is the clock, " + "and includes waiting for retries and rate limits." + ] + if any("cost_usd" not in half for half in halves): + notes.append( + "> **A missing cost is a missing number, not a zero.** The " + "inspect-backed engines report one only where the provider " + "supplies pricing, which a gateway generally does not and which " + "inspect's own registry does not supply for any model; " + "`SKILLSCOPE_MODEL_PRICING` is the way in. Worse for this " + "report: dollars arrive with a run's final event, which a half " + "the bound stopped never reaches -- so the half that spent more " + "is the likelier one to show `n/a`. Tokens and model calls are " + "the like-for-like columns." + ) + if meta.get("concurrency"): + notes.append( + f"> Up to {meta['concurrency']} cases ran at once on each half, " + "so wall time measures how long the run took rather than how " + "much agent time it bought. Model calls and tokens are unaffected." + ) + if any(not half["finished"] for half in halves): + stopped = ", ".join(h["label"] for h in halves if not h["finished"]) + notes.append( + f"> The `{stopped}` row is a floor: the bound stopped it, so it " + "spent at least this and would have spent more." + ) + return notes + + +def _spend_table(halves: list[dict]) -> list[str]: + naming = {"graded": "with the skill", "baseline": "nothing installed"} + rows = [ + "| Half | Wall time | Model calls | Tokens | Cost |", + "| --- | --- | --- | --- | --- |", + ] + for half in halves: + floor = "" if half["finished"] else "+" + rows.append( + f"| {naming.get(half['label'], half['label'])} " + f"| {_duration(half['wall_time_s'])}{floor} " + f"| {_cell(half.get('model_calls'))} " + f"| {_cell(half.get('total_tokens'))} " + f"| {_cell(half.get('cost_usd'))} |" + ) + return rows + + def _detail(check: dict | None) -> str: if check is None: return "not graded this side" @@ -722,6 +853,17 @@ def render_markdown(summary: dict) -> str: "the cases it did not run.", "", ] + # What each half spent. Placed above the verdicts because it is the + # cheaper question and often the more useful answer: a skill whose cases + # all pass either way can still be the difference between two minutes and + # half an hour, and a verdict table alone reports that as "no signal". + # + # Absent from a summary built by something that did not run two halves, + # so `summarize` stays usable for a caller that has only comparisons. + halves = meta.get("legs") or [] + if len(halves) == 2: + lines += _spend_lead(halves) + # Nothing moved at all. The row `earns_it | 0` is true and says nothing, # and this is the one result the command exists to surface, so it is said # rather than left to be noticed in a table. @@ -773,6 +915,10 @@ def render_markdown(summary: dict) -> str: "", ] + if len(halves) == 2: + lines += ["### What each half cost", "", *_spend_table(halves), ""] + lines += [*_spend_caveats(halves, meta), ""] + lines += [ "Reporting only. Agents disagree with themselves between runs, so treat " "a surprising verdict as somewhere to look rather than as a result.", diff --git a/skillscope/usage.py b/skillscope/usage.py index aa70d7f..127a5ca 100644 --- a/skillscope/usage.py +++ b/skillscope/usage.py @@ -17,7 +17,9 @@ from __future__ import annotations -from dataclasses import dataclass +import time +from contextlib import contextmanager +from dataclasses import dataclass, field @dataclass @@ -73,6 +75,76 @@ def record( _current.cost_usd = (_current.cost_usd or 0.0) + float(cost_usd) +@dataclass +class Leg: + """What one half of a paired run spent, and how long it took. + + One object rather than two loose numbers, because both cover exactly the + same interval. That is the only thing that entitles them to share a row. + + `finished` is the caller's to set. A half the deadline stopped reports a + floor rather than a measurement, and a report that cannot tell the two + apart turns "the baseline was still going after half an hour" into "the + baseline took half an hour", which is a weaker claim than the truth. + """ + + label: str + usage: Usage = field(default_factory=Usage) + wall_time_s: float = 0.0 + finished: bool = True + + def as_meta(self) -> dict: + return { + "label": self.label, + "wall_time_s": round(self.wall_time_s, 1), + "finished": self.finished, + **self.usage.as_meta(), + } + + +@contextmanager +def leg(label: str): + """Account one half of a paired run on its own, then fold it into the run. + + The global is *swapped*, not snapshotted and subtracted. `snapshot()` + hands back `_current` itself rather than a copy, so the obvious idiom -- + hold a snapshot, subtract it afterwards -- subtracts the totals from + themselves and reports every half as having spent nothing. That failure + produces plausible zeros rather than an error, which is the kind that + survives review. + + Everything that records does so by name off this module at call time + (`stats.record_log`, `record_stream_event`), so the swap reaches them + without either having to know a leg exists. Nothing records after the + engine returns, and the host leg's stream events run on the event loop + rather than a worker thread, so a scope around the call captures exactly + that half's spend even with several cases in flight. + + Folded back on the way out, so `snapshot()` still answers for the whole + command: the `**usage.snapshot().as_meta()` that every report already + splices stays the sum of the halves, and no existing report changes. + """ + global _current + outer, inner = _current, Usage() + half = Leg(label=label, usage=inner) + _current = inner + started = time.monotonic() + try: + yield half + finally: + half.wall_time_s = time.monotonic() - started + _current = outer + # Through `record` rather than by hand, so one place knows how to add + # one `Usage` into another -- including the rule that an unknown cost + # stays unknown rather than becoming zero. + record( + input_tokens=inner.input_tokens, + output_tokens=inner.output_tokens, + cost_usd=inner.cost_usd, + calls=inner.calls, + ) + + def record_stream_event(event: dict) -> None: """Record what one `claude` stream-json event says the run has spent. diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 15ac75d..cdc1674 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -6321,9 +6321,18 @@ def record(args, skills, gradable, *, install=True, config_dir=None): "cases": [c.id for c in gradable], } ) + # Spend something, and a different something per half, so a test can + # tell which half a figure came from. An engine records from inside + # this call, which is the whole reason the leg scope wraps it. + usage.record( + input_tokens=10 if install else 100, + output_tokens=1 if install else 20, + calls=1 if install else 7, + ) return [] - harness = types.SimpleNamespace(repo=repo, calls=calls) + reports: list[dict] = [] + harness = types.SimpleNamespace(repo=repo, calls=calls, reports=reports) def run(engine="claude-code", extra=None): argv = [ @@ -6336,8 +6345,11 @@ def run(engine="claude-code", extra=None): harness.run = run with mock.patch.object(cli, "_run_behavioral", record), \ - mock.patch.object(cli, "_write_report", lambda *a, **k: None), \ + mock.patch.object( + cli, "_write_report", + lambda summary, *a, **k: reports.append(summary)), \ mock.patch.object(cli, "_prepare_graded_run", lambda *a, **k: ["alpha"]): + usage.reset() yield harness @@ -7550,6 +7562,157 @@ def function(self): self.assertEqual(why, engine_behavioral.UNKNOWN_ROOM) +class TestTheTwoHalvesAreBilledSeparately(unittest.TestCase): + """One global accumulator, two halves, and a trap in how it is read. + + `usage.snapshot()` returns `_current` itself rather than a copy, so the + obvious way to measure a half -- hold a snapshot, subtract it afterwards + -- subtracts the totals from themselves and reports both halves as having + spent nothing. That failure produces plausible zeros rather than an error, + which is the kind nobody catches by reading a report. + + So the scope swaps the global instead, and these pin both halves of that: + each leg sees only its own, and the command total afterwards is still the + sum, which is what keeps every other report's meta unchanged. + """ + + def setUp(self) -> None: + usage.reset() + self.addCleanup(usage.reset) + + def test_each_half_holds_only_its_own_spend(self) -> None: + with usage.leg("graded") as graded: + usage.record(input_tokens=10, output_tokens=1, calls=1) + with usage.leg("baseline") as baseline: + usage.record(input_tokens=100, output_tokens=20, calls=7) + self.assertEqual((graded.usage.input_tokens, graded.usage.calls), (10, 1)) + self.assertEqual((baseline.usage.input_tokens, baseline.usage.calls), (100, 7)) + + def test_the_command_total_is_still_the_sum(self) -> None: + # The no-regression property. `behavioral` and `routing` splice + # `usage.snapshot()` into their own meta and must not notice this. + with usage.leg("graded"): + usage.record(input_tokens=10, calls=1) + with usage.leg("baseline"): + usage.record(input_tokens=100, calls=7) + self.assertEqual(usage.snapshot().input_tokens, 110) + self.assertEqual(usage.snapshot().calls, 8) + + def test_a_snapshot_taken_inside_a_leg_is_that_leg(self) -> None: + # The trap, made explicit. A caller holding this reference and + # subtracting it later gets zero, which is why the scope swaps. + with usage.leg("graded") as graded: + usage.record(input_tokens=10, calls=1) + self.assertIs(usage.snapshot(), graded.usage) + + def test_an_unknown_cost_does_not_fold_back_as_zero(self) -> None: + with usage.leg("graded"): + usage.record(input_tokens=10, calls=1) + self.assertIsNone(usage.snapshot().cost_usd) + + def test_an_engine_recording_from_a_log_lands_in_the_enclosing_half(self) -> None: + # How the real thing records: `stats.record_log` is called deep inside + # the engine, never at the leg boundary, and reaches the global by + # name at call time. That is what the swap relies on. + log = types.SimpleNamespace( + stats=types.SimpleNamespace( + model_usage={"m": types.SimpleNamespace( + input_tokens=42, output_tokens=5, total_cost=None + )} + ), + samples=[], + ) + with usage.leg("baseline") as baseline: + engine_stats.record_log(log) + self.assertEqual(baseline.usage.input_tokens, 42) + + def test_a_half_that_raises_still_folds_back(self) -> None: + # The deadline stops the baseline by unwinding through here. A scope + # that only folded back on success would lose the stopped half's + # spend -- the half most worth knowing the cost of. + with contextlib.suppress(RuntimeError): + with usage.leg("baseline"): + usage.record(input_tokens=100, calls=7) + raise RuntimeError("stopped") + self.assertEqual(usage.snapshot().input_tokens, 100) + + +class TestTheReportSaysWhatEachHalfCost(unittest.TestCase): + """The signal the first version of this command measured and threw away. + + A graded half finished in 2m23s while the baseline was still going half + an hour later, and the report said nothing about it: both halves' cases + passed, so every verdict read `no_signal`. The verdicts cannot express + this, which is why it is said above them. + """ + + @staticmethod + def _summary(*, finished=True, baseline_s=1680.4, cost=None, meta=None): + halves = [ + {"label": "graded", "wall_time_s": 143.2, "finished": True, + "model_calls": 37, "total_tokens": 184220}, + {"label": "baseline", "wall_time_s": baseline_s, + "finished": finished, "model_calls": 214, "total_tokens": 1044901}, + ] + if cost is not None: + for half, value in zip(halves, cost): + half["cost_usd"] = value + return staleness.summarize( + staleness.pair([_outcome("a", True)], [_outcome("a", True)]), + {"legs": halves, **(meta or {})}, + ) + + def _report(self, **kw) -> str: + return staleness.render_markdown(self._summary(**kw)) + + def test_a_baseline_that_was_stopped_is_not_a_baseline_that_was_quick(self) -> None: + report = self._report(finished=False) + self.assertIn("The baseline never finished", report) + self.assertIn("28m00s+", report) + self.assertIn("is a floor", report) + + def test_a_baseline_that_finished_is_compared_as_a_ratio(self) -> None: + report = self._report() + self.assertIn("11.7x the graded half", report) + self.assertNotIn("never finished", report) + + def test_a_skill_that_costs_time_is_said_so_rather_than_hidden(self) -> None: + # A ratio under one is a finding about the skill, not a number to + # suppress because it is unflattering. + report = self._report(baseline_s=30.0) + self.assertIn("The graded half was the slower of the two", report) + + def test_the_spend_is_above_the_verdicts(self) -> None: + report = self._report() + self.assertLess(report.index("What each half cost"), report.index("| Verdict |")) + + def test_an_unknown_cost_reads_as_unknown_and_says_why(self) -> None: + report = self._report() + self.assertIn("| n/a |", report) + self.assertIn("missing cost is a missing number, not a zero", report) + + def test_a_known_cost_is_printed_without_the_caveat(self) -> None: + report = self._report(cost=(0.14, 1.80)) + self.assertIn("0.14", report) + self.assertNotIn("missing cost is a missing number", report) + + def test_running_cases_at_once_is_disclosed_against_the_clock(self) -> None: + # Wall time stops meaning agent time the moment cases overlap, and + # this table is read as though it did. + self.assertIn( + "ran at once on each half", self._report(meta={"concurrency": 4}) + ) + self.assertNotIn("ran at once on each half", self._report()) + + def test_a_summary_with_no_halves_still_renders(self) -> None: + # `summarize` is called by tests and by anything holding only + # comparisons; the spend section is additive and must not be assumed. + report = staleness.render_markdown( + staleness.summarize(staleness.pair([_outcome("a", True)], [_outcome("a", False)]), {}) + ) + self.assertNotIn("What each half cost", report) + + class TestBothHalvesRunAndOnlyTheInstallFlagDiffers(unittest.TestCase): """The command's whole measurement is that the two runs differ in one way. @@ -7576,6 +7739,47 @@ def test_both_halves_grade_the_same_cases(self) -> None: self.assertEqual(harness.calls[0]["cases"], harness.calls[1]["cases"]) self.assertTrue(harness.calls[0]["cases"]) + def test_the_two_halves_are_billed_apart_in_the_report(self) -> None: + # End to end, through the real `cmd_staleness`. The stub spends a + # different amount per half, so a scope that captured both halves + # together -- or captured neither -- shows up here as the two rows + # agreeing. + with _staleness_harness(self) as harness: + harness.run() + legs = harness.reports[0]["meta"]["legs"] + self.assertEqual([half["label"] for half in legs], ["graded", "baseline"]) + self.assertEqual(legs[0]["input_tokens"], 10) + self.assertEqual(legs[1]["input_tokens"], 100) + + def test_the_command_total_still_covers_both_halves(self) -> None: + with _staleness_harness(self) as harness: + harness.run() + meta = harness.reports[0]["meta"] + self.assertEqual(meta["input_tokens"], 110) + self.assertEqual( + meta["input_tokens"], + sum(half["input_tokens"] for half in meta["legs"]), + "the totals are the sum of the halves, or one of them is wrong", + ) + + def test_a_half_the_bound_stopped_is_marked_unfinished(self) -> None: + # `finished` is what the report's floor marker rests on. The graceful + # stage fires before the wall, so `interrupted` is the condition that + # catches a baseline inspect cancelled. + with _staleness_harness(self) as harness: + bound = deadline.Deadline(seconds=600) + bound.interrupted = True + with mock.patch.object(deadline, "active", lambda: bound): + harness.run() + legs = harness.reports[0]["meta"]["legs"] + self.assertFalse(legs[0]["finished"]) + self.assertFalse(legs[1]["finished"]) + + def test_an_unhurried_run_marks_both_halves_finished(self) -> None: + with _staleness_harness(self) as harness: + harness.run() + self.assertTrue(all(half["finished"] for half in harness.reports[0]["meta"]["legs"])) + def test_a_dataset_with_nothing_to_grade_runs_neither_half(self) -> None: with _staleness_harness(self, dataset=tier0_dataset("alpha")) as harness: self.assertEqual(harness.run(), 0) From 72128b18671afee0398112cac66cf59e892195c8 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Thu, 12 Nov 2026 07:15:04 +0100 Subject: [PATCH 06/17] Give each case its own time, on both sides The per-half table says the baseline took twelve times as long. It does not say which cases that went on, which is what an author needs to act. Summed over the attempts that answered, rather than averaged or taken from the decisive attempt. The column has to reconcile with the half-level total above it, and a mean adds up to nothing a reader can check; `_decisive` would be worse, since it prefers an attempt that passed and so selects on the dimension that correlates with speed. `None` rather than `0.0` where a side has no measurement, and that is the whole care in this change. A side whose attempts all errored did not take zero seconds trying: it is the half-hour baseline that never concluded, and rendering it `0s` next to the graded half's `2m23s` states the exact opposite of the finding. The cell says `did not finish`, or `not run` where the case never reached that side at all. --- skillscope/staleness.py | 60 +++++++++++++++++++++++++++++++++++-- tests/test_skillscope.py | 65 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 3 deletions(-) diff --git a/skillscope/staleness.py b/skillscope/staleness.py index df448f0..7a03f32 100644 --- a/skillscope/staleness.py +++ b/skillscope/staleness.py @@ -167,6 +167,12 @@ class Comparison: # can be attempted more than once -- a baseline that was dirty on an # attempt it failed cannot have reached a conclusion it never reached. baseline_dirty_on_pass: bool = False + # Wall seconds each side spent on this case, over the attempts that + # answered. `None` where the side has no honest number -- see `_seconds`, + # where the distinction between "no time" and "no measurement" is the + # difference between reporting the finding and inverting it. + seconds_with: float | None = None + seconds_without: float | None = None def positive(checks: list[dict]) -> list[dict]: @@ -203,6 +209,33 @@ def answered(attempts: list[BehaviorOutcome]) -> list[BehaviorOutcome]: return [a for a in attempts if not a.error] +def _seconds(attempts: list[BehaviorOutcome]) -> float | None: + """Wall seconds this side spent on the case, over the attempts that answered. + + Summed rather than averaged or taken from the decisive attempt. The figure + has to reconcile with the per-half total in the table above it, and a mean + of medians adds up to nothing a reader can check. `_decisive` would be + worse still: it prefers an attempt that *passed*, which selects on exactly + the dimension that correlates with speed, so a baseline's one lucky fast + pass would be reported as its time while three slow failures vanished. At + one attempt a side -- the default -- the sum is that attempt's time. + + The denominator is already on the row: `_passes` prints `3/3 with, 0/3 + without` in the next column, so a reader seeing `7m12s` beside `3/3` reads + it as three attempts' worth without being told. + + `None`, not `0.0`, where no attempt produced a number. This is the one + column where reporting nothing as zero inverts the finding: the case this + command exists for is a baseline that ran for half an hour and never + finished, and `0s` beside the graded half's `2m23s` reads as the baseline + being instant. Zero is discarded for the same reason -- it is what a + synthesised failure carries for a case inspect never ran, and an agent run + is never genuinely instantaneous. + """ + timed = [a.elapsed_s for a in answered(attempts) if a.elapsed_s] + return round(sum(timed), 1) if timed else None + + def _decisive(attempts: list[BehaviorOutcome]) -> BehaviorOutcome | None: """The attempt a side's verdict rests on: one that passed, else the first. @@ -447,6 +480,8 @@ def pair( fired_with=_fired_on_pass(these), baseline_contaminated=_why_dirty(those), baseline_dirty_on_pass=dirty_on_pass(those), + seconds_with=_seconds(these), + seconds_without=_seconds(those), ) ) return comparisons @@ -599,6 +634,22 @@ def _passes(case: dict) -> str: ) +def _seconds_cell(case: dict, side: str) -> str: + """One side's time for one case: a figure, or why there is not one. + + Three states and three strings, because the distinction is the whole + point of the column. A side that errored throughout has no time to show + and did not take zero seconds trying -- it is the half-hour baseline that + never concluded, which is the finding rather than a gap in it. + """ + seconds = case[f"seconds_{side}"] + if seconds is not None: + return _duration(seconds) + if case[f"errored_{side}"]: + return "did not finish" + return "not run" + + def _advice(case: dict) -> str: """What to do about one case, where the verdict alone does not say. @@ -1100,12 +1151,13 @@ def render_markdown(summary: dict) -> str: ) else: lines += [ - "| Case | Verdict | Passed | What to do |", - "| --- | --- | --- | --- |", + "| Case | Verdict | Passed | Time (with / without) | What to do |", + "| --- | --- | --- | --- | --- |", ] for case in notable: lines.append( f"| `{case['id']}` | {case['verdict']} | {_passes(case)} " + f"| {_seconds_cell(case, 'with')} / {_seconds_cell(case, 'without')} " f"| {_advice(case)} |" ) @@ -1133,7 +1185,9 @@ def render_markdown(summary: dict) -> str: ) for case in detailed: lines += [ - f"**`{case['id']}`** ({case['verdict']}) -- {_passes(case)}", + f"**`{case['id']}`** ({case['verdict']}) -- {_passes(case)}; " + f"{_seconds_cell(case, 'with')} with, " + f"{_seconds_cell(case, 'without')} without", "", "| Expectation | With the skill | Without it |", "| --- | --- | --- |", diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index cdc1674..2ff94e3 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -7562,6 +7562,71 @@ def function(self): self.assertEqual(why, engine_behavioral.UNKNOWN_ROOM) +class TestABaselineThatNeverFinishedIsNotABaselineThatWasFast(unittest.TestCase): + """The per-case column, and the one way it can invert its own finding. + + A side whose attempts all errored has no time to report. Writing that as + `0s` beside a graded half's `2m23s` says the baseline was instant, which + is the exact opposite of the case this command was built for: a baseline + that ran for half an hour and never concluded. + """ + + @staticmethod + def _timed(case_id, passed, seconds, *, attempt=1, error=None): + out = _outcome(case_id, passed, attempt=attempt, error=error) + out.elapsed_s = seconds + return out + + def test_a_side_is_the_sum_of_the_attempts_that_answered(self) -> None: + # Summed, so the column reconciles with the per-half total above it. + case = staleness.pair( + [self._timed("c", True, 10.0, attempt=1), + self._timed("c", False, 5.0, attempt=2)], + [self._timed("c", False, 2.0)], + )[0] + self.assertEqual(case.seconds_with, 15.0) + + def test_an_attempt_that_errored_contributes_no_time(self) -> None: + case = staleness.pair( + [self._timed("c", True, 10.0, attempt=1), + self._timed("c", False, 99.0, attempt=2, error="API Error: 504")], + [self._timed("c", False, 2.0)], + )[0] + self.assertEqual(case.seconds_with, 10.0) + + def test_a_side_with_no_time_at_all_is_none_not_zero(self) -> None: + case = staleness.pair( + [self._timed("c", True, 10.0)], + [self._timed("c", False, 0.0, error="API Error: 504")], + )[0] + self.assertIsNone( + case.seconds_without, + "zero seconds and no measurement are different findings here", + ) + + def test_a_side_that_never_finished_says_so_in_the_report(self) -> None: + comparisons = staleness.pair( + [self._timed("c", True, 143.2)], + [self._timed("c", False, 0.0, error="deadline stopped the run")], + ) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("did not finish", report) + self.assertNotIn("0s", report) + + def test_a_case_missing_from_a_side_is_not_run_rather_than_instant(self) -> None: + comparisons = staleness.pair([self._timed("c", True, 10.0)], []) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("not run", report) + + def test_the_times_reach_the_case_table(self) -> None: + comparisons = staleness.pair( + [self._timed("c", True, 143.2)], [self._timed("c", True, 1680.4)] + ) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("2m23s", report) + self.assertIn("28m00s", report) + + class TestTheTwoHalvesAreBilledSeparately(unittest.TestCase): """One global accumulator, two halves, and a trap in how it is read. From ec98388fbf144ee21bd1d39bac6834423f9b4bb9 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Thu, 12 Nov 2026 07:16:25 +0100 Subject: [PATCH 07/17] Document the thinner command The verdict table in `usage.md` listed seven and described a report that no longer exists. The spend comparison is now what the section leads with, since that is what the report leads with and the reason the command is worth running on a suite whose cases all pass. The `no_signal` section says all three findings it now covers, because the old text named only one of them and that one is false for the other two. --- README.md | 2 +- docs/usage.md | 94 +++++++++++++++++++++++++++++++++------------------ 2 files changed, 62 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index afdaea5..b41e21f 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ You may also choose to customize or even disable some tests if you prefer it. Se | `skillscope structural` | Skill folders, datasets, and markdown references. | | `skillscope routing` | Which skill fires, with several installed together. | | `skillscope behavioral` | What a skill does once it has fired. | -| `skillscope staleness` | Whether those cases still need the skill. | +| `skillscope staleness` | Whether those cases still need the skill, and what the agent spends without it. | | `skillscope select` | The CI plan for a change, as JSON. | | `skillscope list-skills` | The skills that have a dataset, as JSON. | | `skillscope template` | The dataset template a new skill starts from. | diff --git a/docs/usage.md b/docs/usage.md index 1b3daef..73ebb27 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -274,45 +274,73 @@ login` is enough for `behavioral` and is not enough here. `--engine claude-code` needs no key for this, since its guest has no user-level skills at all. +The report leads with what each half cost, because that is the cheaper +question and often the more useful answer. An agent holding the skill goes +straight to the work; an agent without it explores. On the first real run of +this command the graded half finished in 2m23s and the baseline was still +going half an hour later, with every case passing on both sides -- a skill +plainly earning its keep, which a verdict table alone reports as `no_signal` +throughout. + +| Half | Wall time | Model calls | Tokens | Cost | +| --- | --- | --- | --- | --- | +| with the skill | 2m23s | 37 | 184220 | n/a | +| nothing installed | 28m00s+ | 214 | 1044901 | n/a | + +A `+` marks a half the `--timeout` stopped: that row is a floor, not a +measurement. Cost reads `n/a` unless the provider supplies pricing, which a +gateway generally does not and which inspect's own registry supplies for no +model at all; `SKILLSCOPE_MODEL_PRICING` is the way in. **A missing cost is a +missing number and not a zero** -- dollars arrive with a run's final event, +which a half the bound stopped never reaches, so the half that spent *more* +is the likelier one to show nothing. Tokens and wall clock are the reliable +columns. + +Then the verdicts. Three, and an error state: + | Verdict | Means | | --- | --- | -| `earns_it` | passed with the skill, failed without it, and the skill fired — the case discriminates | -| `harmful` | failed with the skill, passed without it — the skill steered the agent wrong | -| `unattributed` | passed only with the skill installed, but the skill never fired on the attempt that passed | -| `no_signal` | passed either way — says nothing about the skill | -| `fails_either_way` | failed both times — a broken case, or a hard one | -| `guards_only` | asserts nothing the skill could be credited for — see the guard rails below | -| `ungraded` | nothing could be concluded: every attempt errored on one side or both, a case was missing from a side, its baseline was not empty, or whether the baseline was empty could not be established | +| `earns_it` | passed with the skill, failed without it -- the case discriminates | +| `harmful` | failed with the skill, passed without it -- the skill steered the agent wrong | +| `no_signal` | came out the same both ways | +| `ungraded` | nothing could be concluded: every attempt errored on one side or both, a grader never answered, a case was missing from a side, or the baseline was not the empty room it claimed to be | + +Three rather than more. Earlier versions split out whether the skill fired, +whether a case claimed anything positive at all, and whether a shared failure +was a broken case or a hard one. All three are still reported, as counts and +notes beside the verdict rather than as verdicts of their own. None of them +was a different answer to the question this command asks; each was one of +these answers with a caveat attached. A verdict says what changed, not why. So the graded side also records whether the agent *reached for* the skill, using the same detector the routing leg uses: installing a skill does not make an agent use it, and "installed and the outcome differed" is a weaker claim than "the skill fired and the outcome -differed". A case that moved without the skill firing is `unattributed` rather -than `earns_it`: it stays out of the headline number and is reported as the -lead it is. Guard rails -- `unexpected_behavior` -- are kept -out of the verdict entirely, since a good one holds whether or not the skill is -installed; they are counted on their own, where a guard broken *with* the skill -installed is the serious finding. - -The verdict is binary and the evidence behind it is not, so the report prints -both. Every case carries how many attempts passed on each side, and how much -of the case each side satisfied on average: `fails_either_way` covers a -baseline that met four claims of five and one that met none, and only one of -those says the case is worth keeping. - -`no_signal` is the verdict worth acting on, and it is two findings wearing one -name: either the skill has stopped mattering, or the case was never evidence -about it in the first place. The second is the more common, which makes this a -check on the dataset as much as on the skill. - -Above one attempt there is a third, and the report separates it out. A case -where a side passed some of its attempts and failed others is `no_signal` -because the baseline managed it at least once, and that is not the same -finding as a case both sides managed every single time. One says the agent -does this unaided; the other says the two sides did not settle. The advice -column says which, and the counts beside it are the evidence: act on the -first, re-run the second. +differed". A case that moved without the skill firing is still `earns_it` -- +the outcome did differ -- but it is counted and flagged above the case table, +so the headline share can be discounted by a reader who cares. Guard rails, +`unexpected_behavior`, are kept out of the verdict entirely, since a good one +holds whether or not the skill is installed; they are counted on their own, +where a guard broken *with* the skill installed is the serious finding. + +`no_signal` is the verdict worth acting on, and it is three findings wearing +one name. The pass counts and the advice column say which you are holding: + +* **both sides passed** -- the agent already does this unaided, so the case is + not evidence about the skill. Either the skill has stopped mattering or the + case never tested it, and the second is more common, which makes this as + much a check on the dataset as on the skill. +* **both sides failed** -- a broken case or a hard one. Nothing here has been + shown to work well enough to then ask whether the skill is why, so this + points at the case rather than the skill. +* **a side disagreed with itself** -- only possible above one attempt. The + baseline managed it at least once, which is not the same finding as a case + both sides managed every time. Re-run before reading anything into it. + +A case that claims nothing positive at all is `no_signal` too, since neither +side can meet a claim that was never made. The advice column says so rather +than calling it broken: its guard rails were still checked, and the case is +working exactly as written. ### Spending less on a run @@ -322,7 +350,7 @@ without it explores. Three flags decide what a run costs. | Flag | Default | Effect | | --- | --- | --- | -| `--attempts N` | 1 | Run every case N times on each side. Costs N times as much. A side counts as having passed when **any** of its attempts did, so a higher N is a harder question rather than a steadier reading of the same one: `earns_it` at three attempts needs the baseline to have failed all three times, and a baseline that passes once in three takes the case to `no_signal`. Verdicts compare with verdicts from runs at the same N, not across different ones. What you gain is the counts printed beside each verdict: `3/3` against `0/3` is a result, `3/3` against `1/3` is a baseline that managed it once, and `2/3` against `1/3` is two sides that both disagreed with themselves. The report calls that last one out rather than letting it read as a dead case. | +| `--attempts N` | 1 | Run every case N times on each side. Costs N times as much. A side counts as having passed when **any** of its attempts did, so a higher N is a harder question rather than a steadier reading of the same one: `earns_it` at three attempts needs the baseline to have failed all three times, and a baseline that passes once in three takes the case to `no_signal`. Verdicts compare with verdicts from runs at the same N, not across different ones. What you gain is the counts printed beside each verdict: `3/3` against `0/3` is a result, `3/3` against `1/3` is a baseline that managed it once, and `2/3` against `1/3` is two sides that both disagreed with themselves. The advice column calls that last one out rather than letting it read as a dead case. | | `--max-cases N` | every case | Grade only the first N cases of each selected skill, after `--only` and `--extended` have chosen them. Both sides run the same N, and the report says the run did not cover the whole suite. | | `--concurrency N` | inspect's default | How many cases may be in flight at once. Where the wall clock goes on the baseline half. Raise it as far as the engine's sandboxes and the model's rate limits allow. | From 9ff5277e6dc25e3551af34e12a764c5f12395fe0 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Thu, 12 Nov 2026 08:27:27 +0100 Subject: [PATCH 08/17] Credit a skill for the harm it stopped The first real run of this command found something the report would not say. On `rocm-permission-denied-kfd` the unaided agent attempted a `sudo` write to /opt/rocm/share/rocdxg/dids.conf; the agent holding the skill kept to read-only diagnostics. Both sides failed the case. Every verdict was identical. The only difference between the two runs was the one the report did not mention. Guards were counted in one direction only, so a reader saw "none was broken with the skill installed" and learned that the skill did no harm, never that it stopped any. That is a whole class of a skill's value the pass/fail comparison cannot see: both sides can reach the same answer with only one of them wrecking something on the way. The data was already there -- both sides' guards are graded on every run -- so this is a second `_worst_guards` call, a count, and a sentence. A guard the baseline was never held to is not credit. The baseline has to have been asked the question and failed it; a case it never reached says nothing about what it would have done. Raised by siloteemu on #35, who said outcome comparison could not see this and was right. It turned out not to need the sandbox I claimed it would: the situation reproduced on its own, on the first skill we pointed this at. --- docs/usage.md | 8 ++++- skillscope/staleness.py | 58 ++++++++++++++++++++++++++++++++++++ tests/test_skillscope.py | 64 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 73ebb27..8d8034d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -321,7 +321,13 @@ the outcome did differ -- but it is counted and flagged above the case table, so the headline share can be discounted by a reader who cares. Guard rails, `unexpected_behavior`, are kept out of the verdict entirely, since a good one holds whether or not the skill is installed; they are counted on their own, -where a guard broken *with* the skill installed is the serious finding. +and in both directions. A guard broken *with* the skill installed is the +serious finding. A guard the **baseline** broke and the installed side held +is the opposite one, and the report says so: the skill stopped the agent +doing the forbidden thing. Both sides can reach the same answer with only one +of them wrecking something on the way, and no verdict in the table can +express that. The first real run of this command found one unprompted, an +unaided agent attempting a `sudo` write on a case the skill kept read-only. `no_signal` is the verdict worth acting on, and it is three findings wearing one name. The pass counts and the advice column say which you are holding: diff --git a/skillscope/staleness.py b/skillscope/staleness.py index 7a03f32..fb10c7a 100644 --- a/skillscope/staleness.py +++ b/skillscope/staleness.py @@ -151,6 +151,14 @@ class Comparison: # skill that ran the forbidden thing on an attempt that was not chosen # was reported as having broken nothing at all. guards_with: list[dict] = field(default_factory=list) + # The same, for the baseline. Carried because a guard the *baseline* + # broke and the installed side held is the skill preventing harm, which + # is a kind of value the pass/fail comparison cannot see at all: both + # sides can reach the same answer and only one of them wreck something on + # the way. The first real run of this command produced one -- an unaided + # agent attempting a sudo write to `/opt/rocm/share` on a case the skill + # kept read-only -- and the report credited the skill with nothing. + guards_without: list[dict] = field(default_factory=list) # Whether the agent actually reached for the skill on an attempt the # installed side passed. See `_fired_on_pass`: installing a skill does not # make an agent use one, so an `earns_it` where this is False is a case @@ -290,6 +298,28 @@ def _worst_guards(attempts: list[BehaviorOutcome]) -> list[dict]: return list(worst.values()) +def _harm_prevented(case: dict) -> list[dict]: + """Guards the baseline broke that the installed side held. + + The skill stopped the agent doing something the case forbids. Asked per + expectation rather than per case, because a side can hold one guard and + break another, and it is the specific forbidden thing that a reader needs + named. + + A guard missing from the baseline is not counted. The baseline has to have + been *held to* the guard and failed it; a case the baseline never reached + tells us nothing about what it would have done. + """ + broke = { + g["expectation"]: g for g in case["guards_without"] if not g["passed"] + } + return [ + broke[g["expectation"]] + for g in case["guards_with"] + if g["passed"] and g["expectation"] in broke + ] + + def classify( with_skill: list[BehaviorOutcome], without: list[BehaviorOutcome] ) -> str: @@ -475,6 +505,7 @@ def pair( ) ), guards_with=_worst_guards(these), + guards_without=_worst_guards(those), checks_with=list(here.checks) if here else [], checks_without=list(there.checks) if there else [], fired_with=_fired_on_pass(these), @@ -512,6 +543,12 @@ def summarize(comparisons: list[Comparison], meta: dict) -> dict: if any(not g["passed"] for g in c.guards_with) ] guard_checks = sum(len(c.guards_with) for c in comparisons) + # The other direction, and the one nothing reported. A guard the baseline + # broke and the skill held is the skill earning its place by what it + # stopped the agent doing, which no verdict here can express: both sides + # can reach the same answer with only one of them wrecking something on + # the way. + prevented = [c for c in comparisons if _harm_prevented(asdict(c))] return { "meta": meta, "totals": { @@ -538,6 +575,7 @@ def summarize(comparisons: list[Comparison], meta: dict) -> dict: if c.verdict == "earns_it" and not c.fired_with ), "guard_checks": guard_checks, + "harm_prevented": len(prevented), "guards_broken_with_skill": len(broken), # Counted apart, because they are different findings. One says # the baseline was demonstrably not empty; the other says nobody @@ -1085,6 +1123,26 @@ def render_markdown(summary: dict) -> str: "crediting the skill with them.", ] + saved = [ + (c["id"], g) for c in summary["cases"] for g in _harm_prevented(c) + ] + if saved: + # Said before the guard tally below, which only ever reported guards + # the *skill* broke. A reader who saw "none was broken with the skill + # installed" learned that the skill did no harm, and never that it + # stopped any. + lines += [ + "", + f"> **The skill stopped the agent breaking {len(saved)} guard " + "rail(s) that it broke without it**: " + + "; ".join( + f"`{case_id}` -- {g['expectation'][:80]}" for case_id, g in saved + ) + + ". Both sides may still reach the same answer; this is what the " + "skill prevented on the way, which no verdict in the table above " + "can express.", + ] + if totals["guard_checks"]: broken = totals["guards_broken_with_skill"] lines += [ diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 2ff94e3..7fefa8a 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -6461,6 +6461,70 @@ def __init__(self, sample_id, checks, error=None, epoch=1): self.epoch = epoch +class TestASkillIsCreditedForTheHarmItPrevented(unittest.TestCase): + """A guard the baseline broke and the skill held is the skill working. + + Nothing reported it. The guard tally only ever counted guards the *skill* + broke, so a reader saw "none was broken with the skill installed" and + learned that the skill did no harm, never that it stopped any. + + The first real run of this command produced one unprompted: on + `rocm-permission-denied-kfd` the unaided agent attempted a sudo write to + `/opt/rocm/share/rocdxg/dids.conf` while the agent holding the skill kept + to read-only diagnostics. Both sides failed the case; the verdicts were + identical. The only difference was the one the report did not mention. + """ + + @staticmethod + def _guard(passed: bool, text: str = "no sudo"): + return {"kind": "unexpected_behavior", "expectation": text, + "passed": passed, "detail": "", "graded": True} + + def _case(self, here: bool, there: bool): + return staleness.pair( + [_checked("c", False, [_check("x", True), self._guard(here)])], + [_checked("c", False, [_check("x", True), self._guard(there)])], + )[0] + + def test_a_guard_the_baseline_broke_and_the_skill_held_is_credited(self) -> None: + case = self._case(here=True, there=False) + self.assertEqual(len(staleness._harm_prevented(asdict(case))), 1) + + def test_a_guard_both_sides_held_credits_nothing(self) -> None: + self.assertEqual(staleness._harm_prevented(asdict(self._case(True, True))), []) + + def test_a_guard_the_skill_broke_is_not_credit(self) -> None: + # The opposite finding, and it must not come out as a positive one. + self.assertEqual(staleness._harm_prevented(asdict(self._case(False, True))), []) + + def test_a_guard_neither_side_held_credits_nothing(self) -> None: + self.assertEqual(staleness._harm_prevented(asdict(self._case(False, False))), []) + + def test_a_guard_the_baseline_was_never_held_to_is_not_credit(self) -> None: + # The baseline has to have been asked and failed. A case it never + # reached says nothing about what it would have done, and counting it + # would credit the skill for a comparison that never happened. + case = staleness.pair( + [_checked("c", False, [_check("x", True), self._guard(True)])], + [_checked("c", False, [_check("x", True)])], + )[0] + self.assertEqual(staleness._harm_prevented(asdict(case)), []) + + def test_the_credit_reaches_the_report_and_names_what_was_stopped(self) -> None: + case = self._case(here=True, there=False) + summary = staleness.summarize([case], {}) + self.assertEqual(summary["totals"]["harm_prevented"], 1) + report = staleness.render_markdown(summary) + self.assertIn("stopped the agent breaking", report) + self.assertIn("no sudo", report) + + def test_a_run_with_nothing_prevented_stays_quiet(self) -> None: + report = staleness.render_markdown( + staleness.summarize([self._case(True, True)], {}) + ) + self.assertNotIn("stopped the agent breaking", report) + + class TestGuardRailsAreNotJudgedAsIfTheyDiscriminated(unittest.TestCase): """A guard rail holds with or without the skill -- that is its job. From 7ed54589188d6bdb93844f5443dbba89ecf32e3e Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Thu, 12 Nov 2026 10:20:39 +0100 Subject: [PATCH 09/17] Stop saying the same thing twice about an unsettled case The report carried a paragraph counting `no_signal` cases where a side disagreed with itself, directly above a table whose advice column tells the same reader the same thing on the row they are looking at. The paragraph went; the test moved to the row, which is the better place to assert it. The contamination note keeps its three groups and loses a third of its words. Each group still rests on something different -- a voided verdict claimed the baseline managed something, a standing one rests on the baseline having failed, and the third is dirty on an attempt it failed and clean on the one it passed -- so none of them could be merged without saying something false about one of them. This is a smaller cut than it was meant to be. Going through the report block by block, nearly everything in it either changes how a number should be read or was put there by a bug somebody found. The one genuine duplicate was eleven lines. --- skillscope/staleness.py | 88 +++++++++++++--------------------------- tests/test_skillscope.py | 5 ++- 2 files changed, 33 insertions(+), 60 deletions(-) diff --git a/skillscope/staleness.py b/skillscope/staleness.py index fb10c7a..3e4adc7 100644 --- a/skillscope/staleness.py +++ b/skillscope/staleness.py @@ -1058,58 +1058,44 @@ def render_markdown(summary: dict) -> str: if c["baseline_contaminated"] and c["baseline_contaminated"] != UNKNOWN_ROOM ] if dirty: + # Three groups, because each rests on something different. A verdict + # in `VOIDED_BY_CONTAMINATION` claims the baseline *managed* + # something, so contamination kills it; the others rest on the + # baseline having *failed*, and a baseline that failed while holding + # more than it should have fails all the harder clean. The third + # group is what `--attempts` opened up: dirty on an attempt it + # failed, clean on the one it passed, so the attempt the verdict + # rests on is not the contaminated one. Driven off the constant so + # the rule and the wording cannot drift apart. voided = [c for c in dirty if c["verdict"] == "ungraded"] - # Split three ways, each group worded for what it actually rests on. - # - # The third group is what `--attempts` opened up. `_why_dirty` reads - # every attempt and `dirty_on_pass` reads only the ones that passed, - # so a baseline that was dirty on an attempt it failed and clean on - # the one it passed keeps its verdict. Where that verdict is one of - # `VOIDED_BY_CONTAMINATION` it rests on the baseline having *passed*, - # so the sentence about resting on a failure is false for it. - # - # Driven off `VOIDED_BY_CONTAMINATION` rather than a verdict list - # written out here, so the two cannot drift apart. standing = [c for c in dirty if c["verdict"] != "ungraded"] - stood = [ - c for c in standing - if c["verdict"] not in VOIDED_BY_CONTAMINATION - ] - elsewhere = [ - c for c in standing if c["verdict"] in VOIDED_BY_CONTAMINATION - ] - reasons = sorted({c["baseline_contaminated"] for c in dirty}) - note = [ - "", - f"> **{len(dirty)} case(s) had a baseline that was not empty**: " - + "; ".join(reasons) - + ".", - ] + stood = [c for c in standing if c["verdict"] not in VOIDED_BY_CONTAMINATION] + elsewhere = [c for c in standing if c["verdict"] in VOIDED_BY_CONTAMINATION] + named = lambda group: ", ".join( # noqa: E731 + f"`{c['id']}` ({c['verdict']})" for c in group + ) + lines += ["", f"> **{len(dirty)} case(s) had a baseline that was not " + "empty**: " + "; ".join(sorted( + {c["baseline_contaminated"] for c in dirty})) + "."] if voided: - note.append( + lines.append( f"> {len(voided)} of them are reported `ungraded`: a baseline " - "that reached another skill, or read this skill's own files, " - "cannot show what an agent manages unaided.", + "that reached another skill, or read this skill's own " + "files, cannot show what an agent manages unaided." ) if stood: - note.append( - f"> {len(stood)} still stand -- " - + ", ".join(f"`{c['id']}` ({c['verdict']})" for c in stood) - + " -- because they rest on the baseline *failing*, and it " - "failed while holding more than it should have.", + lines.append( + f"> {len(stood)} still stand ({named(stood)}): they rest on " + "the baseline *failing*, and it failed while holding more " + "than it should have." ) if elsewhere: - note.append( - f"> {len(elsewhere)} case(s) rest on a clean attempt -- " - + ", ".join( - f"`{c['id']}` ({c['verdict']})" for c in elsewhere - ) - + " -- the baseline passed on an attempt that was clean and " - "was dirty on one it failed, so the attempt the verdict rests " - "on is not the contaminated one. They stand, on weaker " - "evidence than a run with no contamination in it at all.", + lines.append( + f"> {len(elsewhere)} rest on a clean attempt " + f"({named(elsewhere)}): the baseline passed on an attempt " + "that was clean and was dirty on one it failed. Weaker " + "evidence than a run with no contamination at all." ) - lines += note unfired = totals["earns_it_unfired"] if unfired: @@ -1162,22 +1148,6 @@ def render_markdown(summary: dict) -> str: c for c in summary["cases"] if c["verdict"] == "no_signal" and _unsettled(c) ] - if unsettled: - lines += [ - "", - f"> **{len(unsettled)} case(s) are `no_signal` because a side " - "disagreed with itself**, not because both sides managed it every " - "time: " - + ", ".join(f"`{c['id']}`" for c in unsettled) - + ". One baseline attempt in three is enough to stop a case " - "being `earns_it` and is not enough to call it dead. Run them " - "again before acting on them.", - ] - - # Former `fails_either_way` cases land here now, because they are - # `no_signal`. That grows the section on a suite with many hard cases, - # which is the right trade: the advice column tells them apart, and this - # is "worth a look" rather than "failures". notable = [ c for c in summary["cases"] if c["verdict"] in ("no_signal", "harmful") diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 7fefa8a..3ecf4e5 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -6776,7 +6776,10 @@ def _report(self, with_results, without_results): def test_a_side_that_disagreed_with_itself_is_called_out(self) -> None: report = self._report([True, True, True], [True, False, False]) - self.assertIn("disagreed with itself", report) + # Asserted on the row rather than on a summary paragraph above it. + # The paragraph said the same thing twice: `_advice` already tells + # this reader, in the column where they are looking at the case. + self.assertIn("the two sides did not settle", report) self.assertIn("did not settle", report) def test_both_sides_managing_it_every_time_keeps_the_old_advice(self) -> None: From a3e3d095cd9094efffc3ce81d4a3176bb3bbba3d Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Fri, 13 Nov 2026 07:54:44 +0100 Subject: [PATCH 10/17] Close the four defects a review pass found **The contamination check failed open on the shapes an agent actually reaches for.** It only recognised a tool call that spelled the skill's own directory, so `find -name SKILL.md`, `grep -r x /`, `cat /*/SKILL.md` and a `..` traversal all read the skill and were reported as a clean room. An agent told nothing and looking for something uses exactly those, which made the miss likeliest on the baseline, the one side whose emptiness this command rests on, and it failed in the dangerous direction: `no_signal` and `harmful` stood where they should have been voided. A `..` traversal is now collapsed textually, which needs no cwd and so is safe here, and names the skill outright. A sweep of a directory the skill sits under is reported as an unknown room instead: the call would take the skill with it and the record cannot prove it did. The ancestor chain stops short of the top level, because a skill checked out under `/tmp/` would otherwise make every `ls /tmp/...` in the run unvouchable. **`_met`'s conjunction was the module's central rule and nothing pinned it.** `all` could become `any` with the whole suite green: a case asserting three things and meeting one would have read as a side that passed, straight into `earns_it`. Every asymmetric test used a single claim, and the only multi-claim tests gave both sides the same pattern, where the two operators cannot disagree. **The scorer glue that makes `judge.Ungradable` mean anything never ran.** `scorers.expectations()` was called by no test, so `graded=False` could be flipped, or the key dropped from `_check` entirely, and nothing failed. The raise was tested and the hand-built dict was tested; the step between them was not. **An ungraded case was choosing the headline.** A suite whose only graded case failed both ways, beside an ungraded case whose graded half happened to pass, was told the skill had stopped earning its place while the row underneath said the case was simply broken. `F841` joins `F401` in CI. It would have caught the dead local this commit removes, left behind when an earlier commit deleted the report section that read it. Every test here was checked against the unfixed code first. --- .github/workflows/selftest.yml | 11 +- skillscope/engine/behavioral.py | 79 +++++++++++++- skillscope/staleness.py | 15 +-- tests/test_skillscope.py | 179 ++++++++++++++++++++++++++++++++ 4 files changed, 273 insertions(+), 11 deletions(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index d87667b..f61811a 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -24,22 +24,27 @@ permissions: jobs: imports: - # One rule, not a linter. `F401` finds imports nothing uses, which is how + # Two rules, not a linter. `F401` finds imports nothing uses, which is how # dead code announces itself: two rounds of review found functions whose # last caller had been deleted, and in both cases the orphaned imports were # sitting there saying so. Twelve of them had accumulated by the time # anybody looked. # + # `F841` is the same signal one scope down, and was added after a review + # found a local left behind by a deleted report section: the list was + # still being built on every run and nothing ever read it. `F401` cannot + # see that, and the whole suite passing cannot either. + # # Deliberately narrow. Adopting a full style configuration is a change to # every file and a decision about house style; this is the one check that # pays for itself immediately and reformats nothing. - name: Unused imports + name: Unused imports and locals runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: astral-sh/ruff-action@v3 with: - args: check --select F401 skillscope tests tools + args: check --select F401,F841 skillscope tests tools unit: name: Unit tests (${{ matrix.os }}, Python ${{ matrix.python }}) diff --git a/skillscope/engine/behavioral.py b/skillscope/engine/behavioral.py index b6a2103..0e1ad5e 100644 --- a/skillscope/engine/behavioral.py +++ b/skillscope/engine/behavioral.py @@ -152,8 +152,77 @@ def _failed(skill: str, cases: list[Case], detail: str) -> list[BehaviorOutcome] def _normalise(text: str) -> str: - """Lowercased with separators flattened, so one comparison covers both OSes.""" - return text.replace("\\", "/").lower() + """Lowercased, separators flattened, `..` segments collapsed. + + The collapse is textual and needs no cwd: `a/b/../c` is `a/c` whoever is + asking. Only a `..` with a real segment in front of it is removed, so a + leading `../x` is left alone -- resolving that *would* need the agent's + cwd, which no tool-call record carries. + + Without it, `/other/../run/SKILL.md` names the skill and matched + nothing, and a baseline that read the skill by that spelling was reported + as a clean room. + """ + flat = text.replace("\\", "/").lower() + if "/../" not in flat: + return flat + out: list[str] = [] + for part in flat.split("/"): + if part == ".." and out and out[-1] not in ("", "..") and ":" not in out[-1]: + out.pop() + continue + out.append(part) + return "/".join(out) + + +# Shapes that read a whole directory rather than one named file. A call +# carrying one of these *and* naming a directory the skill sits in has swept +# the skill up with everything else, without ever spelling its name. +_SWEEPS = ("*", "?", "-r ", "-R ", "--recursive", "find ", "**", "rglob", "ls ") + + +def _swept(text: str, marks: tuple[str, ...]) -> bool: + """Whether this call read a directory containing the skill, wholesale. + + `find -name SKILL.md`, `grep -r x /`, and + `cat /*/SKILL.md` all read the skill and none of them spells its + directory, so matching the skill's own path missed every one. An agent + that has been told nothing and is looking for something reaches for + exactly these, which made the miss likeliest on the baseline -- the one + side whose emptiness the whole command rests on. + + Reported as an unknown room rather than as contamination: the call could + sweep the skill and the record does not prove it did. Unknown and dirty + void the same verdicts, so the conservative answer costs nothing here. + """ + if not any(sweep in text for sweep in _SWEEPS): + return False + return any(_mentions(text, ancestor) for ancestor in _ancestors(marks)) + + +def _ancestors(marks: tuple[str, ...]) -> set[str]: + """Every directory the skill sits under, by its own spelling. + + The whole chain, not just the immediate parent: `find ~/.claude -name + SKILL.md` reads the skill from two levels up and names neither the skill + nor the directory holding it. + + Stops short of the top level, which is where the cost of being wrong + turns over. A skill checked out under `/tmp/` would otherwise + contribute `/tmp` to this set, and then every `ls /tmp/...` in the run -- + including ones nowhere near the skill -- reads as a room nobody can + vouch for. So `/home/u` is kept and `/home` is not. A sweep of a + top-level directory does reach the skill, and flagging it would void + verdicts across whole runs on the strength of a wildcard. + """ + found: set[str] = set() + for mark in marks: + parts = mark.split("/") + while len(parts) > 3: + parts = parts[:-1] + if (parent := "/".join(parts)): + found.add(parent) + return found # Said instead of a contamination reason when there is no way to look for one. @@ -309,6 +378,12 @@ def _read_the_skill(call, marks: tuple[str, ...], skill: str) -> str | None: # the staging area rather than a file of its own. if any("_bundled_skills" in _normalise(v) for v in _strings(arguments)): return reason + + # Last, and weaker on purpose: a call that read a directory the skill + # sits in, wholesale. Answered after the exact matches above so a call + # that provably named the skill keeps the reason that says so. + if any(_swept(_normalise(v), marks) for v in _strings(arguments)): + return UNKNOWN_ROOM return None diff --git a/skillscope/staleness.py b/skillscope/staleness.py index 3e4adc7..37a59f5 100644 --- a/skillscope/staleness.py +++ b/skillscope/staleness.py @@ -964,8 +964,15 @@ def render_markdown(summary: dict) -> str: # has stopped earning its place would point them at the wrong thing # entirely. moved = totals["earns_it"] + totals["harmful"] + # Graded cases only, on both of these. An `ungraded` case is one nobody + # could draw a conclusion from, so letting its counts choose the headline + # lets a case that was never graded speak for the ones that were: a suite + # whose only graded case failed both ways, beside an ungraded case whose + # graded half happened to pass, was told the skill had stopped earning + # its place while the row underneath said the case was simply broken. + settled = [c for c in summary["cases"] if c["verdict"] != "ungraded"] anything_passed = any( - c["passes_with"] or c["passes_without"] for c in summary["cases"] + c["passes_with"] or c["passes_without"] for c in settled ) # A suite that claims nothing positive anywhere is a third case again, # and the one the old `guards_only` verdict used to catch here. Its cases @@ -974,7 +981,7 @@ def render_markdown(summary: dict) -> str: # wording would blame a skill for not affecting checks that never asked # anything of it. claimed = any( - positive(c["checks_with"] + c["checks_without"]) for c in summary["cases"] + positive(c["checks_with"] + c["checks_without"]) for c in settled ) if totals["graded"] and not moved and not claimed: lines += [ @@ -1144,10 +1151,6 @@ def render_markdown(summary: dict) -> str: ), ] - unsettled = [ - c for c in summary["cases"] - if c["verdict"] == "no_signal" and _unsettled(c) - ] notable = [ c for c in summary["cases"] if c["verdict"] in ("no_signal", "harmful") diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 3ecf4e5..259caec 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -68,6 +68,7 @@ from skillscope.engine import no_sandbox as engine_no_sandbox from skillscope.engine import behavioral as engine_behavioral from skillscope.engine import tools as engine_tools +from skillscope.engine import convert as engine_convert from skillscope.engine import scorers as engine_scorers from skillscope.engine import stats as engine_stats from skillscope.engine import pricing as engine_pricing @@ -6525,6 +6526,184 @@ def test_a_run_with_nothing_prevented_stays_quiet(self) -> None: self.assertNotIn("stopped the agent breaking", report) +class TestTheHeadlineIsDrawnFromCasesThatWereGraded(unittest.TestCase): + """An ungraded case was choosing the sentence above the table. + + A suite whose only graded case failed on both sides, sitting beside an + ungraded case whose graded half happened to pass, was told the skill had + "stopped earning its place" -- while the row directly underneath said + "neither side managed it; a broken case or a hard one". The report + contradicted itself, and the half it got wrong was the advice. + """ + + def test_an_ungraded_cases_passes_do_not_choose_the_headline(self) -> None: + comparisons = staleness.pair( + [_outcome("graded-fail", False), _outcome("lost", True)], + [_outcome("graded-fail", False), + _outcome("lost", False, error="API Error: 504")], + ) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("No case in this suite passed on either side", report) + self.assertNotIn("stopped earning its place", report) + + def test_a_suite_that_really_passed_both_ways_still_says_so(self) -> None: + # The control: the other branch has to stay reachable. + comparisons = staleness.pair( + [_outcome("c", True)], [_outcome("c", True)] + ) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("stopped earning its place", report) + + +class TestAnAgentThatWentLookingIsNotACleanRoom(unittest.TestCase): + """The detector only caught a path that spelled the skill's own name. + + Every one of these reads the skill and none of them names its directory. + `find`, `grep -r` and a glob are what an agent reaches for when it has + been told nothing and is looking for something -- so the miss was + likeliest on the baseline, the one side whose emptiness this whole + command rests on, and it failed in the dangerous direction: the room was + reported clean, so `no_signal` and `harmful` stood where they should have + been voided. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill("run", dataset=tier0_dataset("run")) + self.repo.activate() + self.own = str(config.active().skill_path("run")) + + def _why(self, arguments, function="Bash"): + from skillscope.engine import routing as routing_engine + + call = _FakeCall("t", function=function) + call.arguments = arguments + sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) + return engine_behavioral._inspect_room( + routing_engine, sample, "run", self.own + )[1] + + def test_a_parent_traversal_names_the_skill_and_is_contamination(self) -> None: + # Provable: the path resolves to the skill with no cwd needed. + parent, name = self.own.rsplit("/", 1) + why = self._why({"file_path": f"{parent}/other/../{name}/SKILL.md"}, + function="Read") + self.assertIn("read the skill's own files", why or "") + + def test_a_recursive_search_of_a_directory_above_it_is_unknown(self) -> None: + # Not provable, and not clean either: the call would sweep the skill + # up and the record cannot say whether it did. + for command in ( + f"find {self.own.rsplit('/', 1)[0]} -name SKILL.md -exec cat {{}} +", + f"grep -r rocm {self.own.rsplit('/', 1)[0]}/", + f"cat {self.own.rsplit('/', 1)[0]}/*/SKILL.md", + ): + with self.subTest(command=command): + self.assertEqual( + self._why({"command": command}), + engine_behavioral.UNKNOWN_ROOM, + ) + + def test_a_sweep_somewhere_else_entirely_is_still_clean(self) -> None: + # The false positive this must not buy: a sweep is only a sweep of + # *this* skill if it reaches a directory the skill is under. + self.assertIsNone(self._why({"command": "grep -r foo /etc/"})) + self.assertIsNone(self._why({"command": "ls -la /tmp/ws-1/out"})) + self.assertIsNone(self._why({"command": "cat build/*/out.json"})) + + def test_naming_the_skill_outright_still_says_so(self) -> None: + # The sweep answer is weaker than the exact one, so it is asked last. + # A call that provably read the skill keeps the reason that says it. + why = self._why({"command": f"cat {self.own}/SKILL.md"}) + self.assertIn("read the skill's own files", why or "") + + +class TestTheScorerTurnsAnUnansweredJudgeIntoAnUngradedCheck(unittest.TestCase): + """The glue between `judge.Ungradable` and the check it becomes. + + `judge.grade` raising was tested, and a hand-built check dict carrying + `graded: False` was tested, but nothing ran the code that turns the first + into the second. `scorers.expectations()` was called by no test at all, + so `graded=False` could be flipped to `True` -- or the key dropped from + `_check` entirely -- with the whole suite green. The raise is only worth + anything if this converts it. + """ + + def _score(self, grade): + from inspect_ai.scorer import Score # noqa: F401 -- import side effect + + scorer = engine_scorers.expectations() + state = types.SimpleNamespace( + messages=[], metadata={engine_convert.EXPECTED: ["it wrote the file"]}, + output=None, + ) + with mock.patch.object(engine_judge, "grade", grade): + return asyncio.run(scorer(state, None)) + + def test_a_judge_that_could_not_answer_marks_the_check_ungraded(self) -> None: + async def _raise(*a, **k): + raise engine_judge.Ungradable("could not list the sandbox") + + checks = self._score(_raise).metadata[engine_scorers.CHECKS] + self.assertEqual(len(checks), 1) + self.assertFalse(engine_scorers.graded(checks[0])) + self.assertFalse(checks[0]["passed"], "an ungraded check is not a pass") + self.assertIn("could not list the sandbox", checks[0]["detail"]) + + def test_a_judge_that_answered_marks_the_check_graded(self) -> None: + async def _answer(*a, **k): + return True, "judge: it did" + + checks = self._score(_answer).metadata[engine_scorers.CHECKS] + self.assertTrue(engine_scorers.graded(checks[0])) + self.assertTrue(checks[0]["passed"]) + + +class TestASidePassesOnlyByMeetingEveryClaim(unittest.TestCase): + """`_met` is the module's central rule and nothing pinned the conjunction. + + `all(...)` could be changed to `any(...)` and all 763 tests passed. A case + asserting three things and satisfying one would have read as a side that + passed, which flows straight into `earns_it` and the headline share. + + Nothing caught it because every asymmetric test used exactly one claim, + and the only multi-claim tests gave both sides the same mixed pattern -- + where `all` and `any` cannot disagree. This gives the two sides patterns + that the two operators answer differently. + """ + + @staticmethod + def _side(*passes): + return [_checked("c", False, [ + _check(f"claim-{i}", ok) for i, ok in enumerate(passes, 1) + ])] + + def test_meeting_some_claims_is_not_meeting_them(self) -> None: + # 1 of 3 against 0 of 3. Under `any` the graded side "passes" and the + # case is `earns_it`; under `all` neither side passed. + verdict = staleness.classify( + self._side(True, False, False), self._side(False, False, False) + ) + self.assertEqual( + verdict, "no_signal", + "a side that met one claim of three has not met every claim", + ) + + def test_meeting_every_claim_is(self) -> None: + # The control, same shape, so the pair can only be told apart by the + # operator under test. + verdict = staleness.classify( + self._side(True, True, True), self._side(False, False, False) + ) + self.assertEqual(verdict, "earns_it") + + def test_the_counts_follow_the_same_rule(self) -> None: + case = staleness.pair( + self._side(True, False, False), self._side(False, False, False) + )[0] + self.assertEqual((case.passes_with, case.passes_without), (0, 0)) + + class TestGuardRailsAreNotJudgedAsIfTheyDiscriminated(unittest.TestCase): """A guard rail holds with or without the skill -- that is its job. From 9f5db2caf3745fd6a98df67f09d98eefb1e866d4 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Fri, 13 Nov 2026 09:13:20 +0100 Subject: [PATCH 11/17] Build the room-check test's paths the way Windows spells them The new contamination tests split the skill's path on "/" to reach its parent directory. Windows hands back `C:\...\run`, where that split finds no separator and returns the whole path, so one test raised and three asserted against the skill's own directory instead of the directory above it. The code under test normalises separators itself and was right on both runners; only the test assumed POSIX. `Path.parent` asks the question in a way that holds either way. Confirmed by running a Windows-shaped path through `_normalise`, `_ancestors` and `_swept` directly: the ancestor chain comes out at the same depth as the POSIX one, because the leading empty segment on POSIX and the drive letter on Windows occupy the same slot. --- tests/test_skillscope.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 259caec..16612a0 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -6585,18 +6585,28 @@ def _why(self, arguments, function="Bash"): def test_a_parent_traversal_names_the_skill_and_is_contamination(self) -> None: # Provable: the path resolves to the skill with no cwd needed. - parent, name = self.own.rsplit("/", 1) - why = self._why({"file_path": f"{parent}/other/../{name}/SKILL.md"}, - function="Read") + # + # Built through `Path` rather than by splitting on "/". Windows hands + # back `C:\\...\\run`, where a `rsplit("/")` finds no separator and + # returns the whole path -- so an earlier version of this test raised + # on one runner and asserted against the skill's own directory on the + # other. The code under test normalises separators itself; the test + # has to be told. + own = Path(self.own) + why = self._why( + {"file_path": f"{own.parent}/other/../{own.name}/SKILL.md"}, + function="Read", + ) self.assertIn("read the skill's own files", why or "") def test_a_recursive_search_of_a_directory_above_it_is_unknown(self) -> None: # Not provable, and not clean either: the call would sweep the skill # up and the record cannot say whether it did. + above = Path(self.own).parent for command in ( - f"find {self.own.rsplit('/', 1)[0]} -name SKILL.md -exec cat {{}} +", - f"grep -r rocm {self.own.rsplit('/', 1)[0]}/", - f"cat {self.own.rsplit('/', 1)[0]}/*/SKILL.md", + f"find {above} -name SKILL.md -exec cat {{}} +", + f"grep -r rocm {above}/", + f"cat {above}/*/SKILL.md", ): with self.subTest(command=command): self.assertEqual( From 3f0dc369a816e880072622148797748fa65e1564 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Mon, 16 Nov 2026 07:58:39 +0100 Subject: [PATCH 12/17] Close the six a second review pass found **The ancestor chain never ran for the staged skill.** `_ancestors` stopped above three segments, a floor calibrated for the absolute mark. The staged one is `/` -- `.claude/skills/`, relative, exactly three -- so it contributed no ancestors at all and every sweep of the staged directory read as a clean room. Ran against the production code: `ls .claude/skills/`, `find .claude -name SKILL.md` and `grep -r x .claude/skills/` all came back `None`. The floor is now a property of the mark. An absolute path spends a segment on the empty string before the root and stops at `/home/u`; the staged mark keeps every segment, because it is relative to a workspace made for one sample and each of its segments is the harness's own. `/home` holds every user on the machine, `.claude` holds skills and nothing else of ours. **A `no_signal` report told a baseline it had passed when it had passed nothing.** `no_signal` is two findings under one name, and the both-sides- failed one reaches the contamination paragraph too: `dirty_on_pass` fires only on an attempt that passed, so a baseline dirty on the one attempt it had, and failing it, is never voided and landed in the group whose sentence reads "the baseline passed on an attempt that was clean and was dirty on one it failed". It passed on no attempt at all. Split on `passes_without`, which counts on the same basis `dirty_on_pass` reads. **A staleness report was the one graded report that did not name its harness.** `cmd_staleness` hand-wrote its meta where `cmd_behavioral` and `_finish_routing` splice `_leg_meta`, so it carried no `harness` or `harness_version` and spelled the engine without `_engine_label`'s normalisation. **`-R ` in `_SWEEPS` was dead the day it was written.** Every caller passes `_normalise(...)`, which lowercases, so only `-r ` could ever match. **A test that starts no agent started one.** `test_the_default_harness_is_ not_refused` passes `--harness claude-code`, which by design does not hit the refusal -- and the next thing `cmd_staleness` does is `_prepare_graded_ run`, which probes model reachability and spawns the real `claude` CLI whenever credentials are in the environment. Its class is meant to touch no network and every sibling test is refused before that call. Stopped at the preflight instead, which proves the guard did not fire *and* that execution reached it. **`attempt` was derived in one line nothing covered.** `_StubSample` was built for exactly that and never instantiated; every multi-attempt test constructs outcomes by hand with `attempt=i`, so the line could have been hard-coded to 1 with the suite green -- and collapsing it collides every repeat in `_no_duplicates`. Each fix is pinned by a test that fails with the fix reverted: the floor mutated back to a constant 3, the split dropped, `_leg_meta` unspliced, the guard widened to refuse the default, and `attempt` hard-coded to 1. 780 tests pass, ruff `F401,F841` clean. --- skillscope/cli.py | 9 ++- skillscope/engine/behavioral.py | 30 +++++++- skillscope/staleness.py | 26 ++++++- tests/test_skillscope.py | 129 +++++++++++++++++++++++++++++++- 4 files changed, 188 insertions(+), 6 deletions(-) diff --git a/skillscope/cli.py b/skillscope/cli.py index 45da9ca..61a7aa4 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -1088,8 +1088,15 @@ def cmd_staleness(args: argparse.Namespace) -> int: summary = staleness.summarize( comparisons, { + # Which agent graded, where it ran, and which build of it, the + # way every other graded report says it. Hand-written before, + # which spelled the engine without `_engine_label`'s + # normalisation and left out `harness` and `harness_version` + # altogether -- a reader holding a routing or behavioral report + # beside a staleness one found two keys missing and a third in + # a different alphabet. + **_leg_meta(args), "model": args.model, - "engine": args.engine, "effort": args.effort, "skills": skills, "extended": args.extended, diff --git a/skillscope/engine/behavioral.py b/skillscope/engine/behavioral.py index 0e1ad5e..5d2b299 100644 --- a/skillscope/engine/behavioral.py +++ b/skillscope/engine/behavioral.py @@ -178,7 +178,10 @@ def _normalise(text: str) -> str: # Shapes that read a whole directory rather than one named file. A call # carrying one of these *and* naming a directory the skill sits in has swept # the skill up with everything else, without ever spelling its name. -_SWEEPS = ("*", "?", "-r ", "-R ", "--recursive", "find ", "**", "rglob", "ls ") +# Lowercase only: every caller passes `_normalise(...)`, so an upper-case +# spelling here can never match. `-R ` sat in this tuple reading as +# case-sensitive coverage and was dead the day it was written. +_SWEEPS = ("*", "?", "-r ", "--recursive", "find ", "**", "rglob", "ls ") def _swept(text: str, marks: tuple[str, ...]) -> bool: @@ -214,11 +217,34 @@ def _ancestors(marks: tuple[str, ...]) -> set[str]: vouch for. So `/home/u` is kept and `/home` is not. A sweep of a top-level directory does reach the skill, and flagging it would void verdicts across whole runs on the strength of a wildcard. + + Where that top level is depends on the mark, and the two marks do not + have one in common. A single floor of three was calibrated for the + absolute mark and left the staged one -- `/`, three + segments on every registered harness -- with no ancestors at all, so + `ls .claude/skills/`, `find .claude -name SKILL.md` and `grep -r x + .claude/skills/` all came back as a clean room. That is the shape an + agent told nothing reaches for, on the one side whose emptiness the + whole command rests on. + + So the staged mark keeps every segment and the absolute one does not. + The floor is not about depth, it is about what else lives there: `/home` + holds every user on the machine and `/tmp` holds every run, while the + staged mark is relative to a workspace made for one sample and each of + its segments is the harness's own -- `.claude` holds skills and nothing + else of ours. A sweep of it reaches the skill and almost nothing a + baseline would touch for another reason, and the answer it buys is + `UNKNOWN_ROOM` rather than contamination, so the conservative reading + costs a verdict nobody could vouch for anyway. """ found: set[str] = set() for mark in marks: parts = mark.split("/") - while len(parts) > 3: + # Absolute (`/x/...`, or a drive letter after `_normalise`) spends a + # segment on the empty string before the root, so three there and + # one here leave `/home/u` and `.claude` respectively. + floor = 3 if (parts[0] == "" or ":" in parts[0]) else 1 + while len(parts) > floor: parts = parts[:-1] if (parent := "/".join(parts)): found.add(parent) diff --git a/skillscope/staleness.py b/skillscope/staleness.py index 37a59f5..19bde05 100644 --- a/skillscope/staleness.py +++ b/skillscope/staleness.py @@ -1077,7 +1077,24 @@ def render_markdown(summary: dict) -> str: voided = [c for c in dirty if c["verdict"] == "ungraded"] standing = [c for c in dirty if c["verdict"] != "ungraded"] stood = [c for c in standing if c["verdict"] not in VOIDED_BY_CONTAMINATION] - elsewhere = [c for c in standing if c["verdict"] in VOIDED_BY_CONTAMINATION] + # The third group splits again on whether the baseline ever passed. + # `no_signal` is two findings under one name, and the both-sides- + # failed one reaches here too: `dirty_on_pass` fires only on an + # attempt that passed, so a baseline dirty on an attempt it failed + # is never voided and lands beside the cases that did pass clean. + # Told as one group, it informed a case whose baseline passed + # nothing at all that it had passed on a clean attempt. + # `passes_without` counts on the same basis `dirty_on_pass` reads -- + # `_met` over `answered` -- so zero there means there was no passing + # attempt for contamination to have landed on. + elsewhere = [ + c for c in standing + if c["verdict"] in VOIDED_BY_CONTAMINATION and c["passes_without"] + ] + never_passed = [ + c for c in standing + if c["verdict"] in VOIDED_BY_CONTAMINATION and not c["passes_without"] + ] named = lambda group: ", ".join( # noqa: E731 f"`{c['id']}` ({c['verdict']})" for c in group ) @@ -1103,6 +1120,13 @@ def render_markdown(summary: dict) -> str: "that was clean and was dirty on one it failed. Weaker " "evidence than a run with no contamination at all." ) + if never_passed: + lines.append( + f"> {len(never_passed)} still stand ({named(never_passed)}): " + "the baseline passed nothing on any attempt, so no verdict " + "here rests on something it managed while holding more than " + "it should have." + ) unfired = totals["earns_it_unfired"] if unfired: diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 16612a0..34d9e2d 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -36,7 +36,7 @@ import unittest from dataclasses import asdict import urllib.error -from pathlib import Path +from pathlib import Path, PurePosixPath from unittest import mock import skillscope @@ -6462,6 +6462,50 @@ def __init__(self, sample_id, checks, error=None, epoch=1): self.epoch = epoch +class TestARepeatIsTakenFromTheSampleThatRanIt(unittest.TestCase): + """`attempt` has to come off the sample's own epoch, not a constant. + + Every other multi-attempt test builds `BehaviorOutcome`s by hand with + `attempt=i`, so the one line in production that derives the field -- + `attempt=int(getattr(sample, "epoch", 1) or 1)` -- could have been + hard-coded to 1 or read the wrong attribute with the suite still green. + That line is what keeps `--attempts` repeats of one case apart: collapse + it and every repeat collides in `_no_duplicates`, and `_by_case` has one + attempt where the run had three. + """ + + @staticmethod + def _case(case_id: str): + return datasets.Case( + id=case_id, skill="alpha", prompt="p", skill_should_trigger=True, + ) + + @staticmethod + def _checks(): + return [{"kind": "expected_behavior", "expectation": "e", + "passed": True, "detail": "", "graded": True}] + + def test_each_repeat_carries_its_own_number(self) -> None: + log = types.SimpleNamespace( + status="success", + error=None, + samples=[_StubSample("c1", self._checks(), epoch=n) + for n in (1, 2, 3)], + ) + outcomes = engine_behavioral._outcomes(log, "alpha", [self._case("c1")]) + self.assertEqual([o.attempt for o in outcomes], [1, 2, 3]) + + def test_a_sample_with_no_epoch_is_the_first_attempt(self) -> None: + # Ordinary single-pass runs, and any log written before epochs were + # asked for, have to read as attempt 1 rather than 0 -- `_by_case` + # and `_no_duplicates` both key on it. + sample = _StubSample("c1", self._checks()) + del sample.epoch + log = types.SimpleNamespace(status="success", error=None, samples=[sample]) + outcome = engine_behavioral._outcomes(log, "alpha", [self._case("c1")])[0] + self.assertEqual(outcome.attempt, 1) + + class TestASkillIsCreditedForTheHarmItPrevented(unittest.TestCase): """A guard the baseline broke and the skill held is the skill working. @@ -6627,6 +6671,40 @@ def test_naming_the_skill_outright_still_says_so(self) -> None: why = self._why({"command": f"cat {self.own}/SKILL.md"}) self.assertIn("read the skill's own files", why or "") + def test_a_sweep_of_the_staged_copy_is_unknown_too(self) -> None: + # The other mark, and the one every test above missed. The staged + # copy is `/` -- `.claude/skills/run`, relative, + # three segments -- and `_ancestors` stopped above three, so it + # contributed nothing and every sweep of the staged directory read + # as a clean room. The absolute mark these other tests use is deep + # enough never to reach that edge. + # + # `no_sandbox` runs the agent with `cwd=workspace`, so these are the + # spellings an agent standing in one actually types. + from skillscope import harness as harnesses + + skills_dir = harnesses.get().skills_dir + for command in ( + f"ls {skills_dir}/", + f"find {PurePosixPath(skills_dir).parts[0]} -name SKILL.md", + f"grep -r rocm {skills_dir}/", + f"cat {skills_dir}/*/SKILL.md", + ): + with self.subTest(command=command): + self.assertEqual( + self._why({"command": command}), + engine_behavioral.UNKNOWN_ROOM, + ) + + def test_the_top_of_the_machine_is_still_the_floor(self) -> None: + # What the floor is for, unchanged: the absolute mark stops short of + # the directory the whole machine lives under, because counting that + # would void verdicts across whole runs on the strength of a + # wildcard nowhere near the skill. + top = Path(self.own).parts[1] + self.assertIsNone(self._why({"command": f"ls -la /{top}/"})) + self.assertIsNone(self._why({"command": f"grep -r foo /{top}"})) + class TestTheScorerTurnsAnUnansweredJudgeIntoAnUngradedCheck(unittest.TestCase): """The glue between `judge.Ungradable` and the check it becomes. @@ -7175,7 +7253,23 @@ def test_the_default_harness_is_not_refused(self) -> None: # was ever reached -- so it asserted the absence of a refusal that # could not have fired, and would have gone on passing had the guard # been written to refuse the default too. - self.assertNotIn("no staleness leg", self._run("--harness", "claude-code")) + # + # Stopped at the preflight rather than let through it. Past the + # guard the next thing `cmd_staleness` does is `_prepare_graded_run`, + # which probes model reachability and spawns the real `claude` CLI + # whenever credentials happen to be in the environment -- which they + # are on a developer machine and on a credentialed runner. This class + # is meant to start no agent and touch no network, and every sibling + # test is refused before reaching that call. Raising from the stub + # proves the guard did not fire *and* that execution got as far as + # the preflight. + with mock.patch.object( + cli, "_prepare_graded_run", + side_effect=SystemExit("error: preflight reached"), + ): + message = self._run("--harness", "claude-code") + self.assertNotIn("no staleness leg", message) + self.assertIn("preflight reached", message) def _pair(self, with_results, without_results): return staleness.pair( @@ -7317,6 +7411,24 @@ def test_a_verdict_resting_on_a_clean_attempt_is_not_called_a_failure(self) -> N self.assertIn("rest on a clean attempt", report) self.assertNotIn("rest on the baseline *failing*", report) + def test_a_baseline_that_passed_nothing_is_not_told_it_passed(self) -> None: + # The same paragraph, the other way wrong. `no_signal` covers both + # sides *failing* as well as both passing, and `dirty_on_pass` fires + # only on an attempt that passed -- so a baseline dirty on the one + # attempt it had, and failing it, is never voided and landed in the + # group whose sentence says the baseline "passed on an attempt that + # was clean". It passed on no attempt at all. One side one attempt: + # `--attempts` is not needed to reach this. + comparisons = staleness.pair( + [_outcome("c", False)], + [self._baseline(False, "another skill fired: beta")], + ) + self.assertEqual(comparisons[0].verdict, "no_signal") + self.assertEqual(comparisons[0].passes_without, 0) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertNotIn("rest on a clean attempt", report) + self.assertIn("passed nothing on any attempt", report) + class TestEarningItIsReportedWithWhetherTheSkillFired(unittest.TestCase): """Installing a skill does not make an agent use it. @@ -8101,6 +8213,19 @@ def test_an_unhurried_run_marks_both_halves_finished(self) -> None: harness.run() self.assertTrue(all(half["finished"] for half in harness.reports[0]["meta"]["legs"])) + def test_the_report_names_the_harness_like_every_other_one(self) -> None: + # `cmd_staleness` hand-wrote its meta, so a staleness report was the + # one graded report with no `harness` or `harness_version` in it, and + # with the engine spelled without `_engine_label`'s normalisation. + # A reader holding this beside a routing or behavioral report found + # two keys missing and a third in a different alphabet. + with _staleness_harness(self) as harness: + harness.run() + meta = harness.reports[0]["meta"] + self.assertEqual(meta["harness"], cli.harnesses.DEFAULT) + self.assertIn("harness_version", meta) + self.assertEqual(meta["engine"], "claude-code") + def test_a_dataset_with_nothing_to_grade_runs_neither_half(self) -> None: with _staleness_harness(self, dataset=tier0_dataset("alpha")) as harness: self.assertEqual(harness.run(), 0) From fbd4a034f59c2f5207f7a8a72466e7d501770dce Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Mon, 16 Nov 2026 12:56:44 +0100 Subject: [PATCH 13/17] Catch the spellings an agent actually types, and pin the branches Two blocking findings from the third review pass. Both are about the same thing from opposite ends: the baseline's emptiness is the whole measurement, and neither the check that establishes it nor the branches around it were holding. **The contamination check matched the tidy spelling of each shape only.** Probed against the code, each of these came back a clean room: `grep -ri foo /`, `grep -rn`, `ls -lR`, `rg`, `tree`, `fd`, `cat ~/`, `cat $HOME/`, `//` and `/./`. `-r ` was a literal, and a flag an agent almost never types alone. It is now any single-dash cluster containing `r`, which is what `-ri`, `-rn`, `-nr` and `-lR` all are; `rg`, `tree`, `fd` and `grep` join `find` and `ls`, and the command words are matched on a word boundary rather than as substrings, so a sweep is a command somebody ran rather than two letters inside another word. `~` was expanded only for an argument that was a bare path with no whitespace in it -- and a Bash command line is never that, which is the only place an agent types one. `~/`, `$HOME/` and `${HOME}/` are now written out wherever they appear, anchored so they only fire where a path can start. The expansion resolves against this process's home, which is the agent's own on the live path and a guess on a rebuild; the comment claiming otherwise is corrected rather than left standing. `_normalise` collapsed `/../` but not `//` or `/./`, which a shell joining two pieces of a path produces without anybody typing them. All three are now collapsed, along with a trailing separator, which costs nothing because `_mentions` already treats the end of the string as a boundary. **The branches nothing would have caught.** Each of these was mutated alone and the suite stayed green; each now has a test that fails when it is: - `_inspect_room`'s two guards -- a later `other:` hit overwriting the first reason, and a later firing replacing the first. These are the exact bug the function's own docstring says it fixed, and no test ever walked more than one activating call. - `scorers.files_exist`'s `graded=False` on a sandbox that would not list. Flipping it to `True` turns a harness failure back into the agent's, and a baseline whose sandbox would not list manufactures `earns_it`. - the judge's `must_happen` polarity. Hard-wired to `True`, every `unexpected_behavior` check inverts: an agent that stayed away from the forbidden thing is reported as having done it. - `_met`'s `bool(claims)`. Without it `all([])` makes a case that asserts nothing read as a side that passed. - `_decisive`'s preference for a passing attempt that fired, which is what keeps `fired_with` and `checks_with` describing the same attempt. - `_why_dirty`'s preference for a known reason over `UNKNOWN_ROOM`, which is compared as a sentinel downstream and so decides which group a case is reported in. - `_seconds` discarding a zero, the one column where reporting nothing as zero inverts the finding. - `_run_behavioral`'s `install`, `attempts`, `concurrency`, `baseline` and `config_dir`, which were asserted by grepping the function's source. The stub engine calls `solver_factory` the way the real one does, because `install` and `config_dir` reach the agent only through that closure. 802 tests pass, ruff `F401,F841` clean. --- skillscope/engine/behavioral.py | 143 ++++++++---- tests/test_skillscope.py | 391 ++++++++++++++++++++++++++++++++ 2 files changed, 495 insertions(+), 39 deletions(-) diff --git a/skillscope/engine/behavioral.py b/skillscope/engine/behavioral.py index 5d2b299..9411a27 100644 --- a/skillscope/engine/behavioral.py +++ b/skillscope/engine/behavioral.py @@ -16,6 +16,7 @@ from __future__ import annotations +import re import sys from pathlib import Path @@ -152,22 +153,36 @@ def _failed(skill: str, cases: list[Case], detail: str) -> list[BehaviorOutcome] def _normalise(text: str) -> str: - """Lowercased, separators flattened, `..` segments collapsed. + """Lowercased, separators flattened, no-op segments collapsed. The collapse is textual and needs no cwd: `a/b/../c` is `a/c` whoever is - asking. Only a `..` with a real segment in front of it is removed, so a - leading `../x` is left alone -- resolving that *would* need the agent's - cwd, which no tool-call record carries. + asking, and so are `a//c` and `a/./c`. Only a `..` with a real segment in + front of it is removed, so a leading `../x` is left alone -- resolving + that *would* need the agent's cwd, which no tool-call record carries. Without it, `/other/../run/SKILL.md` names the skill and matched nothing, and a baseline that read the skill by that spelling was reported - as a clean room. + as a clean room. `//` and `/./` are the same miss in a cheaper spelling: + a shell that builds a path by joining two pieces produces them without + anyone typing them, and both name the skill exactly. + + A trailing separator goes with them, which costs nothing: `_mentions` + treats the end of the string as a boundary, so `/run/` and + `/run` match the same mark. """ flat = text.replace("\\", "/").lower() - if "/../" not in flat: + # `/.` covers `/./`, `/../` and a trailing `/.` or `/..`; `//` covers a + # repeated separator anywhere. + if "//" not in flat and "/." not in flat: return flat out: list[str] = [] - for part in flat.split("/"): + for index, part in enumerate(flat.split("/")): + # A repeated or trailing separator. The leading empty segment of an + # absolute path is kept: it is what makes the path absolute. + if part == "" and index and out: + continue + if part == "." and out: + continue if part == ".." and out and out[-1] not in ("", "..") and ":" not in out[-1]: out.pop() continue @@ -175,22 +190,75 @@ def _normalise(text: str) -> str: return "/".join(out) +# `~/x` and `$HOME/x` spell the same file as the absolute path, and a mark is +# always absolute. Anchored so it only fires where a path can start, and only +# where one actually follows. +_HOME = re.compile(r"(? str: + """`text` again with `~/` and `$HOME/` written out, where they appear. + + Separate from `_normalise` because it is the one expansion that is not + context-free. It resolves against *this* process's home, which is the + agent's own on the live path -- `no_sandbox` runs it on this host -- and + is a guess on a rebuild of logs written somewhere else. Worth making + anyway: the alternative is reading `cat ~/dev/x/skills/run/SKILL.md` as + a clean room, and the answer it buys is voided-or-unknown rather than a + claim about the skill. + + Applied to whole command strings, not just to arguments that are paths. + The expansion before this one fired only on a value that started with + `~` and had no whitespace in it, which is a bare path -- so every `~` + inside a Bash command line, which is where an agent actually types one, + went unexpanded. + """ + if "~" not in text and "$" not in text: + return text + try: + home = _normalise(str(Path.home())) + except (Exception, SystemExit): # noqa: BLE001 -- no home, nothing to do + return text + return _HOME.sub(home, text) if home else text + + # Shapes that read a whole directory rather than one named file. A call # carrying one of these *and* naming a directory the skill sits in has swept # the skill up with everything else, without ever spelling its name. -# Lowercase only: every caller passes `_normalise(...)`, so an upper-case -# spelling here can never match. `-R ` sat in this tuple reading as -# case-sensitive coverage and was dead the day it was written. -_SWEEPS = ("*", "?", "-r ", "--recursive", "find ", "**", "rglob", "ls ") +# +# Globs, as substrings: they are not words and can sit anywhere in a path. +_SWEEP_GLOBS = ("*", "?", "**", "rglob") + +# The rest are words, and matching them as substrings was the bug twice over. +# `-r ` is a flag an agent almost never types alone: `grep -ri`, `grep -rn` +# and `ls -lR` all sweep and none of them contains `-r ` -- so the shapes the +# docstring below names as the realistic ones were matched only in their +# tidiest spelling. The flag is now any single-dash cluster containing `r`, +# which is what those all are, and the commands are matched on a word +# boundary so a sweep is a command somebody ran rather than two letters +# inside another word. +# +# Bare `grep` is deliberately not here. `grep foo ` does not read the +# directory -- it is the recursive flag or a glob that does, and both are +# matched already -- so listing it would void verdicts on the strength of a +# command that never reached the skill. +# +# Lowercase only, deliberately: every caller passes `_normalise(...)`, so an +# upper-case spelling here can never match. `-R ` sat in the old tuple +# reading as case-sensitive coverage and was dead the day it was written; +# `\w*r\w*` covers it now because the text is lowercased first. +_SWEEP_WORDS = re.compile( + r"(?:^|\s)(?:-\w*r\w*|--recursive|find|ls|rg|tree|fd)(?=\s|$)" +) def _swept(text: str, marks: tuple[str, ...]) -> bool: """Whether this call read a directory containing the skill, wholesale. - `find -name SKILL.md`, `grep -r x /`, and - `cat /*/SKILL.md` all read the skill and none of them spells its - directory, so matching the skill's own path missed every one. An agent - that has been told nothing and is looking for something reaches for + `find -name SKILL.md`, `grep -ri x /`, `rg x ` + and `cat /*/SKILL.md` all read the skill and none of them spells + its directory, so matching the skill's own path missed every one. An + agent that has been told nothing and is looking for something reaches for exactly these, which made the miss likeliest on the baseline -- the one side whose emptiness the whole command rests on. @@ -198,7 +266,7 @@ def _swept(text: str, marks: tuple[str, ...]) -> bool: sweep the skill and the record does not prove it did. Unknown and dirty void the same verdicts, so the conservative answer costs nothing here. """ - if not any(sweep in text for sweep in _SWEEPS): + if not any(glob in text for glob in _SWEEP_GLOBS) and not _SWEEP_WORDS.search(text): return False return any(_mentions(text, ancestor) for ancestor in _ancestors(marks)) @@ -364,10 +432,18 @@ def _strings(value) -> list[str]: def _read_the_skill(call, marks: tuple[str, ...], skill: str) -> str | None: """Whether this call reached into the skill's own directory. - Every string argument is tested as written, and a `~`-relative one is - tested expanded as well. Expansion is skipped for anything with whitespace - in it: that is a command line rather than a path, and it is matched as - written. + Every string argument is tested as written and again with `~/` and + `$HOME/` written out, wherever in it they appear. The expansion before + this one fired only on a value that started with `~` and had no + whitespace -- a bare path -- so a `~` inside a Bash command line, which + is where an agent actually types one, was never expanded at all. + + `./x` and `../x` are deliberately *not* resolved against a cwd. The agent's + was `no_sandbox`'s per-sample workspace and appears nowhere in the + tool-call record, so `resolve()` here resolved them against skillscope's + own instead and made the verdict depend on which directory the report was + run from. (`_normalise` does collapse an *interior* `/../` or `/./`, + which needs no cwd to be right.) The graded side reads its own installed skill by design, so a hit there is expected. It is the baseline where this means the room was not empty. @@ -377,38 +453,27 @@ def _read_the_skill(call, marks: tuple[str, ...], skill: str) -> str | None: return None reason = f"read the skill's own files ({skill})" + spellings: list[str] = [] for value in _strings(arguments): text = _normalise(value) + spellings.append(text) + if (expanded := _expand_home(text)) != text: + spellings.append(expanded) + + for text in spellings: if any(_mentions(text, mark) for mark in marks): return reason - # `~/x` names the same file by another spelling, and expanding it is - # context-free: `$HOME` is the agent's own, recorded in the - # environment rather than inferred from where this is running. - # - # `./x` and `../x` are deliberately *not* resolved. Resolving them - # needs the agent's cwd, which is `no_sandbox`'s per-sample workspace - # and appears nowhere in the tool-call record -- so `resolve()` here - # resolved them against skillscope's own cwd instead, and made the - # verdict depend on which directory the report was run from. It never - # could serve the case it was written for. - if value.startswith("~") and not any(c.isspace() for c in value): - try: - expanded = _normalise(str(Path(value).expanduser())) - except (Exception, SystemExit): # noqa: BLE001 -- not a path, then - continue - if any(_mentions(expanded, mark) for mark in marks): - return reason # The one mark that is not a path. `_bundled_skills` is the harness's own # staging directory, so its name in any argument is the agent having found # the staging area rather than a file of its own. - if any("_bundled_skills" in _normalise(v) for v in _strings(arguments)): + if any("_bundled_skills" in text for text in spellings): return reason # Last, and weaker on purpose: a call that read a directory the skill # sits in, wholesale. Answered after the exact matches above so a call # that provably named the skill keeps the reason that says so. - if any(_swept(_normalise(v), marks) for v in _strings(arguments)): + if any(_swept(text, marks) for text in spellings): return UNKNOWN_ROOM return None diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 34d9e2d..65fd5c2 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -6706,6 +6706,202 @@ def test_the_top_of_the_machine_is_still_the_floor(self) -> None: self.assertIsNone(self._why({"command": f"grep -r foo /{top}"})) +class TestWhatTheHalvesAskTheEngineFor(unittest.TestCase): + """`_run_behavioral`'s kwargs were asserted by reading its source. + + `install`, `attempts`, `concurrency`, `baseline` and `config_dir` are + what make the baseline half a baseline, and the only test touching them + grepped the function for engine names or replaced the function with a + mock. Each could be dropped, or wired to the wrong value, with the suite + green -- and `baseline=not install` inverted is the whole measurement + reported backwards. + """ + + @staticmethod + def _args(engine): + return argparse.Namespace( + engine=engine, model="mockllm/model", effort="high", + attempts=3, concurrency=4, harness=None, + ) + + @staticmethod + @contextlib.contextmanager + def _host_leg(seen: dict): + """The host engine and its harness, recorded rather than run. + + The engine's `run` calls `solver_factory` the way the real one does, + because `install` and `config_dir` reach the agent only through that + closure -- a stub that never calls it records a solver nobody built + and the two arguments that make the baseline a baseline go unseen. + """ + no_sandbox = types.SimpleNamespace( + require_local=lambda subject: None, + harness_solver=lambda *a, **k: seen.update(solver=k, built=a) or "s", + ) + + def run(*a, **k): + seen.update(k) + k["solver_factory"]("dataset") + return [] + + with mock.patch.object( + engine_module, "no_sandbox", no_sandbox + ), mock.patch.object( + engine_module, "behavioral", types.SimpleNamespace(run=run) + ): + yield + + def test_the_host_leg_is_told_which_half_it_is(self) -> None: + seen: dict = {} + with self._host_leg(seen): + cli._run_behavioral( + self._args("claude-code-no-sandbox"), ["alpha"], [], + install=False, config_dir=Path("/tmp/bare"), + ) + self.assertTrue(seen["baseline"], "the half with no skill is the baseline") + self.assertEqual(seen["attempts"], 3) + self.assertEqual(seen["concurrency"], 4) + self.assertFalse(seen["solver"]["install"]) + self.assertIn( + Path("/tmp/bare"), seen["built"], + "the bare config dir is what keeps this machine's own skills out", + ) + + def test_the_graded_half_is_not_the_baseline(self) -> None: + # The control. `baseline=not install` reads the same either way until + # both halves are run through it. + seen: dict = {} + with self._host_leg(seen): + cli._run_behavioral( + self._args("claude-code-no-sandbox"), ["alpha"], [], install=True, + ) + self.assertFalse(seen["baseline"]) + self.assertTrue(seen["solver"]["install"]) + + def test_the_sandboxed_leg_carries_the_same_three(self) -> None: + seen = {} + verify = types.SimpleNamespace(run=lambda *a, **k: seen.update(k) or []) + with mock.patch.object(engine_module, "verify", verify): + cli._run_behavioral( + self._args("claude-code"), ["alpha"], [], install=False, + ) + self.assertFalse(seen["install"]) + self.assertEqual(seen["attempts"], 3) + self.assertEqual(seen["concurrency"], 4) + + +class TestTheRecursiveShapesAnAgentActuallyTypes(unittest.TestCase): + """The sweep detector matched the tidiest spelling of each shape only. + + `_SWEEPS` held the literal `"-r "`, so `grep -ri`, `grep -rn` and `ls -lR` + -- the flags an agent actually types -- all swept the skill up and were + reported as a clean room. `rg`, `tree` and `fd` were not listed at all. + `~` was expanded only for an argument that was a bare path, so a `~` + inside a Bash command line, which is the only place one appears, was + never expanded. And `_normalise` collapsed `/../` but not `//` or + `/./`, which a shell joining two pieces of a path produces without + anybody typing them. + + Every one of these was probed against the code and came back `None`. + This is the failure class the sweep detector was added to close, on the + one side whose emptiness the command rests on. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill("run", dataset=tier0_dataset("run")) + self.repo.activate() + self.own = Path(config.active().skill_path("run")) + + def _why(self, command): + from skillscope.engine import routing as routing_engine + + call = _FakeCall("t", function="Bash") + call.arguments = {"command": command} + sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) + return engine_behavioral._inspect_room( + routing_engine, sample, "run", str(self.own) + )[1] + + def test_a_recursive_flag_in_any_cluster_is_a_sweep(self) -> None: + above = self.own.parent + for command in ( + f"grep -ri foo {above}/", + f"grep -rn foo {above}/", + f"grep -Rn foo {above}/", + f"ls -lR {above}", + f"grep -nr foo {above}", + ): + with self.subTest(command=command): + self.assertEqual(self._why(command), engine_behavioral.UNKNOWN_ROOM) + + def test_the_other_search_tools_sweep_too(self) -> None: + above = self.own.parent + for command in (f"rg foo {above}", f"tree {above}", f"fd SKILL.md {above}"): + with self.subTest(command=command): + self.assertEqual(self._why(command), engine_behavioral.UNKNOWN_ROOM) + + def test_a_flag_with_no_r_in_it_is_not_a_sweep(self) -> None: + # The false positive the cluster rule must not buy: a single-dash + # flag is only a sweep when it actually asks for recursion. + above = self.own.parent + self.assertIsNone(self._why(f"head -n 5 {above}/notes.txt")) + self.assertIsNone(self._why(f"cat -A {above}/notes.txt")) + + @contextlib.contextmanager + def _home_at(self, where: Path): + """`Path.home()` answering `where`, which is where the skill is. + + The repo fixture puts the skill in a temp directory rather than under + the real home, so without this the `~` spellings resolve nowhere near + it and the test passes by asserting on an absence. Patched rather + than skipped: a skipped test pins nothing, and this is the branch + that was reported missing. + """ + with mock.patch.object( + engine_behavioral.Path, "home", classmethod(lambda cls: where) + ): + yield + + def test_a_home_relative_path_inside_a_command_is_expanded(self) -> None: + # The shape an agent types. The expansion before this fired only on + # an argument that was a bare `~`-path with no whitespace, and a + # command line is never that, so `~` was never expanded at all. + tail = self.own.name + with self._home_at(self.own.parent): + for spelling in ( + f"cat ~/{tail}/SKILL.md", + f"cat $HOME/{tail}/SKILL.md", + f"cat ${{HOME}}/{tail}/SKILL.md", + ): + with self.subTest(spelling=spelling): + self.assertIn( + "read the skill's own files", self._why(spelling) or "" + ) + + def test_a_home_relative_sweep_is_expanded_too(self) -> None: + with self._home_at(self.own.parent.parent): + self.assertEqual( + self._why(f"find ~/{self.own.parent.name} -name SKILL.md"), + engine_behavioral.UNKNOWN_ROOM, + ) + + def test_a_doubled_or_dotted_separator_still_names_the_skill(self) -> None: + parent, name = self.own.parent, self.own.name + for spelling in ( + f"cat {parent}//{name}/SKILL.md", + f"cat {parent}/./{name}/SKILL.md", + f"cat {parent}//.//{name}/SKILL.md", + ): + with self.subTest(spelling=spelling): + self.assertIn("read the skill's own files", self._why(spelling) or "") + + def test_a_tilde_belonging_to_something_else_is_left_alone(self) -> None: + # The anchor on the expansion: `~` is a home only where a path can + # start. A backup file's suffix is not one. + self.assertIsNone(self._why("cat notes.txt~/x")) + + class TestTheScorerTurnsAnUnansweredJudgeIntoAnUngradedCheck(unittest.TestCase): """The glue between `judge.Ungradable` and the check it becomes. @@ -6746,6 +6942,126 @@ async def _answer(*a, **k): self.assertTrue(engine_scorers.graded(checks[0])) self.assertTrue(checks[0]["passed"]) + def test_a_forbidden_expectation_is_graded_on_its_own_polarity(self) -> None: + # `must_happen` is what tells the grader which answer is the failure. + # Hard-wiring it to True left the suite green and inverts every + # `unexpected_behavior` check: the grader is asked whether the + # forbidden thing happened and the answer is read as whether the + # required thing did, so an agent that stayed away from it is + # reported as having done it. + asked: list[bool] = [] + + async def _answer(statement, state, *, must_happen): + asked.append(must_happen) + return True, "judge: it did" + + from inspect_ai.scorer import Score # noqa: F401 -- import side effect + + scorer = engine_scorers.expectations() + state = types.SimpleNamespace( + messages=[], + metadata={ + engine_convert.EXPECTED: ["it wrote the file"], + engine_convert.UNEXPECTED: ["it ran sudo"], + }, + output=None, + ) + with mock.patch.object(engine_judge, "grade", _answer): + asyncio.run(scorer(state, None)) + self.assertEqual(asked, [True, False]) + + +class TestASandboxThatWouldNotListIsNotTheAgentsFault(unittest.TestCase): + """`files_exist` has the same two-step as the judge and nothing ran it. + + `tools.list_paths` raising reports every wanted path as a check that + failed *and was not graded*. Flipping that `graded=False` to `True` left + the whole suite green -- and it is the one edit that turns a harness + failure back into the agent's: `staleness.answered` keeps the attempt, + the side reads as failing, and a baseline whose sandbox would not list + manufactures `earns_it` for the skill. + """ + + def _score(self, lister): + from inspect_ai.scorer import Score # noqa: F401 -- import side effect + + scorer = engine_scorers.expectations() + state = types.SimpleNamespace( + messages=[], metadata={engine_convert.FILES_EXIST: ["out.txt"]}, + output=None, + ) + with mock.patch.object(engine_tools, "list_paths", lister): + return asyncio.run(scorer(state, None)) + + def test_a_listing_that_failed_leaves_the_check_ungraded(self) -> None: + async def _raise(): + raise engine_tools.ListingFailed("no container") + + checks = self._score(_raise).metadata[engine_scorers.CHECKS] + self.assertEqual(len(checks), 1) + self.assertFalse(engine_scorers.graded(checks[0])) + self.assertFalse(checks[0]["passed"], "an ungraded check is not a pass") + self.assertIn("could not list the sandbox", checks[0]["detail"]) + + def test_a_listing_that_worked_grades_the_check(self) -> None: + # The other direction, and the one a careless fix breaks: a sandbox + # that listed and simply does not hold the file is the agent's + # failure and has to stay graded. + async def _empty(): + return [] + + checks = self._score(_empty).metadata[engine_scorers.CHECKS] + self.assertTrue(engine_scorers.graded(checks[0])) + self.assertFalse(checks[0]["passed"]) + + +class TestTheRoomWalkKeepsTheFirstAnswerItFound(unittest.TestCase): + """Two questions, one walk, and each must keep its own first answer. + + The branches exist because they shared an `if/elif` once: a second + `other:` hit arriving after `why` was set fell through to the `elif` and + was recorded as *the skill* having fired, which reads downstream as the + skill activating on exactly the contaminated cases that most need the + opposite. Both guards could be dropped with the suite green, because no + test ever walked more than one activating call. + """ + + @staticmethod + def _sample(hits): + calls = [_FakeCall(f"t{i}") for i, _ in enumerate(hits)] + for call in calls: + call.arguments = {"command": "echo hello"} + return _FakeSample(messages=[_FakeMessage("m", calls)], events=[]), hits + + def _walk(self, hits): + from skillscope.engine import routing as routing_engine + + sample, answers = self._sample(hits) + answers = list(answers) + with mock.patch.object( + engine_behavioral.routing_core, "detect_activation", + lambda *a, **k: answers.pop(0), + ): + return engine_behavioral._inspect_room( + routing_engine, sample, "run", "/nowhere/skills/run" + ) + + def test_a_later_contamination_does_not_replace_the_first_reason(self) -> None: + fired, why = self._walk(["other:alpha", "other:beta"]) + self.assertIsNone(fired) + self.assertEqual(why, "another skill fired: alpha") + + def test_a_contaminating_hit_is_never_recorded_as_the_skill_firing(self) -> None: + # The regression itself: the skill fires, then another skill does. + # `fired` has to stay the skill and `why` has to be set. + fired, why = self._walk(["run", "other:beta"]) + self.assertEqual(fired, "run") + self.assertEqual(why, "another skill fired: beta") + + def test_the_first_firing_is_the_one_kept(self) -> None: + fired, _ = self._walk(["run", "run-two"]) + self.assertEqual(fired, "run", "a later hit must not replace the first") + class TestASidePassesOnlyByMeetingEveryClaim(unittest.TestCase): """`_met` is the module's central rule and nothing pinned the conjunction. @@ -6791,6 +7107,81 @@ def test_the_counts_follow_the_same_rule(self) -> None: )[0] self.assertEqual((case.passes_with, case.passes_without), (0, 0)) + def test_a_side_asserting_nothing_has_not_met_anything(self) -> None: + # The other half of the conjunction. `all([])` is True, so dropping + # `bool(claims)` makes a case that asserts no positive claim at all + # read as a side that passed -- on *both* sides, since neither has + # claims, which is how it stays quiet. A case carrying only a guard + # rail is exactly that shape, and the module has a name for what it + # is worth: nothing to credit. + guard = {"kind": "unexpected_behavior", "expectation": "no sudo", + "passed": True, "detail": ""} + nothing_asserted = _checked("c", False, [guard]) + self.assertFalse(staleness._met(nothing_asserted)) + self.assertEqual( + staleness.classify([nothing_asserted], [_checked("c", False, [guard])]), + "no_signal", + ) + + +class TestTheEvidenceComesOffTheAttemptTheVerdictRestsOn(unittest.TestCase): + """Three one-line preferences, each of which could be dropped unnoticed. + + They decide which attempt's evidence the report prints, which reason it + gives for a dirty baseline, and whether a side's time is a figure or an + absence. All three mattered on the first real run of this command and + none of them was pinned. + """ + + def test_a_passing_attempt_that_fired_is_preferred(self) -> None: + # `_fired_on_pass` asks whether the skill fired on any attempt that + # passed; this is what keeps `fired_with` and `checks_with` pointing + # at that same attempt. Drop it and the report shows the checks of an + # attempt where nothing fired beside a note saying something did -- + # two different runs described as one. + quiet = _checked("c", True, [_check("x", True)], activated=None, attempt=1) + fired = _checked("c", True, [_check("x", True)], activated="s", attempt=2) + self.assertIs(staleness._decisive([quiet, fired]), fired) + + def test_a_passing_attempt_still_beats_a_failing_one(self) -> None: + # The control: the firing preference must not overtake the passing + # one, or a side that passed is reported off an attempt that failed. + failed = _checked("c", False, [_check("x", False)], activated="s", attempt=1) + passed = _checked("c", True, [_check("x", True)], activated=None, attempt=2) + self.assertIs(staleness._decisive([failed, passed]), passed) + + def test_a_known_reason_beats_one_nobody_could_establish(self) -> None: + # They are different findings: one says the baseline demonstrably + # held something, the other that nobody could tell. A side with one + # attempt of each has the answer to the second in the first, so + # reporting "could not be checked" throws away the finding -- and + # `UNKNOWN_ROOM` is compared as a sentinel downstream, so the swap + # changes which group the case is reported in, not just its wording. + unknown = _outcome("c", False) + unknown.contaminated = engine_behavioral.UNKNOWN_ROOM + known = _outcome("c", False, attempt=2) + known.contaminated = "another skill fired: beta" + self.assertEqual( + staleness._why_dirty([unknown, known]), "another skill fired: beta" + ) + self.assertEqual( + staleness._why_dirty([unknown]), engine_behavioral.UNKNOWN_ROOM, + "with nothing better it is still reported", + ) + + def test_a_side_that_recorded_no_time_reports_none_not_zero(self) -> None: + # The one column where reporting nothing as zero inverts the finding. + # Zero is what a synthesised failure carries for a case inspect never + # ran, and `0s` beside the graded half's `2m23s` reads as the + # baseline having been instant -- the exact opposite of the + # half-hour baseline this command exists to surface. + never_ran = _checked("c", False, [_check("x", False)]) + never_ran.elapsed_s = 0.0 + self.assertIsNone(staleness._seconds([never_ran])) + ran = _checked("c", False, [_check("x", False)]) + ran.elapsed_s = 12.5 + self.assertEqual(staleness._seconds([never_ran, ran]), 12.5) + class TestGuardRailsAreNotJudgedAsIfTheyDiscriminated(unittest.TestCase): """A guard rail holds with or without the skill -- that is its job. From 5e52fa32776803fe801f2c38cd8026e69137f568 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Tue, 17 Nov 2026 12:55:35 +0100 Subject: [PATCH 14/17] Say which half ran out of time, and make four tests able to fail **A baseline that never started was reported as the slow one.** The halves share one `--timeout`, so a graded half that spends all of it leaves the baseline none: every baseline case returns failed without an agent ever starting. `_spend_lead` read only the baseline's `finished` flag, so the headline said "the baseline was still going 0s in, when the bound stopped it ... an agent without the skill reaches it only by exploring" -- a half that never ran, described as exactly the finding this command exists to produce. The graded half's flag is what tells the two states apart, and there is now a fourth sentence for the one where the budget ran out first. `test_a_half_the_bound_stopped_is_marked_unfinished` built this state already and asserted only the flags, so the sentence went unread; it is now asserted, with a control for the genuine case -- graded half finished, baseline stopped partway -- that must keep saying what it said. **Three tests asserted the absence of strings the code never emits.** `assertNotIn("disagreed with itself", ...)` and `assertNotIn("Every graded case either", ...)` match nothing in `skillscope/`; the code says "did not settle" and "Every graded case needed the skill". One of them, `test_one_attempt_a_side_can_never_read_as_unsettled`, had no other assertion, so it could never fail. Each now names the string the code prints, and the one that was alone has gained the positive half. **And one of mine from last round.** The tilde test used `cat notes.txt~/`, which reads as a clean room whether the anchor on `_HOME` is there or not: expanding it puts the skill's path straight after `notes.txt`, and `_mentions` rejects a mark that starts mid-word. For a word character the boundary check is what blocks the false positive and the anchor is belt and braces. The test now uses `$$HOME/` and `~~/`, where what precedes the expansion is itself a boundary and the anchor is the only thing between the spelling and a hit. Both go red with the lookbehind removed; the old one did not. **Docs corrected to what the code does.** `staleness` refuses any `--harness` other than `claude-code` rather than refusing `--harness` outright -- `--harness claude-code` is accepted, and a test pins that. Fixed in `README.md`, `docs/usage.md` and `docs/harnesses.md`. The README's CI paragraph now names the three graders `reusable.yml` actually runs and says `staleness` is not among them. 804 tests pass, ruff `F401,F841` clean. The new branch and each repaired test were checked by mutation. --- README.md | 7 ++-- docs/harnesses.md | 3 +- docs/usage.md | 6 +-- skillscope/staleness.py | 27 ++++++++++++-- tests/test_skillscope.py | 80 ++++++++++++++++++++++++++++++++++++---- 5 files changed, 105 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b41e21f..10f4a8b 100644 --- a/README.md +++ b/README.md @@ -54,12 +54,13 @@ credential question on Claude Code. Its baseline half has to be a room with **no** skills in it, so the runner's own are not noise there — they are the measurement. Either set one of those variables, or use `--engine claude-code`, whose guest has no user-level skills at all. It is Claude Code -only: unlike `routing`, it refuses `--harness` outright rather than per -harness, because there is no staleness leg for another one yet. +only: unlike `routing`, which scales its refusal per harness, it refuses +any `--harness` other than `claude-code`, because there is no staleness leg +for another one yet. ## Run it from your CI -Simply add an `evals.json` file to your skill and a workflow that points to our reusable workflow. This will triger all graders to run directly from your repo. +Simply add an `evals.json` file to your skill and a workflow that points to our reusable workflow. This will trigger the three graders the workflow runs -- `structural`, `routing` and `behavioral` -- directly from your repo. `staleness` is not one of them: it runs both halves of every case and is meant to be asked for deliberately rather than on every push. Skillscope diff --git a/docs/harnesses.md b/docs/harnesses.md index a8ad233..3d70e0c 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -148,4 +148,5 @@ Claude Code, and a harness other than it cannot yet use them: than a harness's own `skills_dir`; * **`staleness`**: its baseline is only a measurement if the room is empty, and the check that establishes that reads Claude Code's credentials and no other - harness's, so it refuses `--harness` outright rather than per harness. + harness's, so it refuses any `--harness` other than `claude-code` rather + than scaling the refusal per harness. diff --git a/docs/usage.md b/docs/usage.md index 8d8034d..103973e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -448,9 +448,9 @@ routing leg at all unless one of the harness's credential variables is set config dir away from the runner's own. `staleness` is stricter, and not on the same terms. It has no leg for any -harness but Claude Code yet, so it refuses `--harness` outright whatever -credentials are present, rather than scaling the refusal per harness the way -routing does. On Claude Code it then asks routing's question as well, because +harness but Claude Code yet, so it refuses any `--harness` other than +`claude-code` whatever credentials are present, rather than scaling the +refusal per harness the way routing does. On Claude Code it then asks routing's question as well, because there the empty room is not one input to the measurement, it is the measurement. Refusing beats being quietly wrong about every case. diff --git a/skillscope/staleness.py b/skillscope/staleness.py index 19bde05..769c6c1 100644 --- a/skillscope/staleness.py +++ b/skillscope/staleness.py @@ -767,12 +767,31 @@ def _spend_lead(halves: list[dict]) -> list[str]: against a baseline still going half an hour later, and the report said nothing about it. - Three shapes, because the honest sentence differs. A baseline that was - stopped is a floor and is said as one. A graded half slower than its - baseline is a finding and is said plainly rather than suppressed: a skill - that costs the agent time is something its author wants to know. + Four shapes, because the honest sentence differs. A baseline that was + stopped is a floor and is said as one -- but only where it ran at all. A + graded half slower than its baseline is a finding and is said plainly + rather than suppressed: a skill that costs the agent time is something + its author wants to know. + + The halves share one bound, so the graded half can spend all of it and + leave the baseline none: every baseline case then returns failed without + an agent ever starting. Both halves are marked unfinished, which is true, + and reading only the baseline's flag told the reader it "was still going + 0s in" -- a baseline that never ran, reported as the slow exploring one + this command exists to find. The graded half's flag is what tells the two + apart. """ graded, baseline = halves + if not graded["finished"]: + return [ + "> **The bound ran out before the baseline could start.** The " + f"graded half used the whole {_duration(graded['wall_time_s'])} " + "and was stopped partway, so there is no baseline to compare it " + "against -- the cases below are reported against a half that " + "never ran. Raise `--timeout`, or cut the run down with " + "`--max-cases`, and read the verdicts only after that.", + "", + ] if not baseline["finished"]: return [ f"> **The baseline never finished.** The graded half took " diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 65fd5c2..8a49828 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -6896,10 +6896,30 @@ def test_a_doubled_or_dotted_separator_still_names_the_skill(self) -> None: with self.subTest(spelling=spelling): self.assertIn("read the skill's own files", self._why(spelling) or "") - def test_a_tilde_belonging_to_something_else_is_left_alone(self) -> None: - # The anchor on the expansion: `~` is a home only where a path can - # start. A backup file's suffix is not one. - self.assertIsNone(self._why("cat notes.txt~/x")) + def test_a_home_marker_that_is_not_one_is_left_alone(self) -> None: + # The anchor on the expansion: `~` or `$HOME` is a home only where a + # path can start, not where it continues a word. + # + # The command has to be one the expansion would turn *into* a hit if + # the anchor came off, or the test passes either way -- which is how + # the first version of this was written. `cat notes.txt~/` + # reads as clean with the anchor or without it, because expanding it + # puts the skill's path straight after `notes.txt` and `_mentions` + # rejects a mark that starts mid-word. For the word-character case + # the boundary check is what blocks the false positive and the + # anchor is belt and braces; these two are where the anchor is the + # only thing standing between the spelling and a hit, because what + # precedes the expansion is itself a boundary character. + with self._home_at(self.own.parent): + for command in ( + f"cat $$HOME/{self.own.name}/SKILL.md", # a PID, then a word + f"cat ~~/{self.own.name}/SKILL.md", + ): + with self.subTest(command=command): + self.assertIsNone( + self._why(command), + "a home marker continuing another token is not a home", + ) class TestTheScorerTurnsAnUnansweredJudgeIntoAnUngradedCheck(unittest.TestCase): @@ -7441,12 +7461,21 @@ def test_a_side_that_disagreed_with_itself_is_called_out(self) -> None: self.assertIn("did not settle", report) def test_both_sides_managing_it_every_time_keeps_the_old_advice(self) -> None: + # "disagreed with itself" is the sentence this class is named for and + # appears nowhere in the code, so asserting its absence asserted + # nothing. The string the code emits is "did not settle". report = self._report([True, True], [True, True]) - self.assertNotIn("disagreed with itself", report) + self.assertNotIn("did not settle", report) self.assertIn("already did this unaided", report) def test_one_attempt_a_side_can_never_read_as_unsettled(self) -> None: - self.assertNotIn("disagreed with itself", self._report([True], [True])) + # One attempt a side cannot disagree with itself, so the unsettled + # advice must not fire -- and the positive assertion is what makes + # this test capable of failing at all: without it the only check was + # for a string nothing emits. + report = self._report([True], [True]) + self.assertNotIn("did not settle", report) + self.assertIn("already did this unaided", report) class TestAReportWithNothingGradedAssertsNothingAboutGradedCases(unittest.TestCase): @@ -7469,7 +7498,12 @@ def test_a_guards_only_run_is_not_told_its_graded_cases_came_out_well(self) -> N [_checked("c", True, [self._guard(True)])], [_checked("c", True, [self._guard(True)])], )) - self.assertNotIn("Every graded case either", report) + # The string the code emits is "Every graded case needed the skill"; + # "Every graded case either" appears nowhere in it, so asserting the + # absence of that one asserted nothing at all. The positive + # assertion below is what this test rests on either way -- it is the + # sentence the branch under test prints. + self.assertNotIn("Every graded case needed the skill", report) self.assertIn("Every case here only forbids things", report) def test_a_broken_guard_is_not_followed_by_nothing_to_look_at(self) -> None: @@ -8599,6 +8633,38 @@ def test_a_half_the_bound_stopped_is_marked_unfinished(self) -> None: self.assertFalse(legs[0]["finished"]) self.assertFalse(legs[1]["finished"]) + def test_a_baseline_that_never_started_is_not_called_the_slow_one(self) -> None: + # The state above, read. The two halves share one bound, so a graded + # half that spends all of it leaves the baseline none: every baseline + # case returns failed without an agent starting. `_spend_lead` read + # only the baseline's flag, so the headline said it "was still going + # 0s in" -- a half that never ran, reported as the slow exploring one + # this whole command exists to find. Asserting the flags alone left + # that green, which is why this asserts the sentence. + with _staleness_harness(self) as harness: + bound = deadline.Deadline(seconds=600) + bound.interrupted = True + with mock.patch.object(deadline, "active", lambda: bound): + harness.run() + report = staleness.render_markdown(harness.reports[0]) + self.assertIn("before the baseline could start", report) + self.assertNotIn("The baseline never finished", report) + self.assertNotIn("was still going 0s in", report) + + def test_a_baseline_the_bound_stopped_partway_is_still_a_floor(self) -> None: + # The control, and the sentence that must survive: the graded half + # finished, the baseline was stopped mid-run. That is the half-hour + # baseline the command was written for, and it is still said as one. + with _staleness_harness(self) as harness: + harness.run() + summary = harness.reports[0] + graded, baseline = summary["meta"]["legs"] + graded.update(finished=True, wall_time_s=143.0) + baseline.update(finished=False, wall_time_s=1800.0) + report = staleness.render_markdown(summary) + self.assertIn("The baseline never finished", report) + self.assertNotIn("before the baseline could start", report) + def test_an_unhurried_run_marks_both_halves_finished(self) -> None: with _staleness_harness(self) as harness: harness.run() From 00b6b21fd4145d8233d04caa91da2514ffd8535b Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Tue, 17 Nov 2026 13:09:43 +0100 Subject: [PATCH 15/17] Build the sandboxed baseline in a test, rather than grepping for it The `claude-code` leg's baseline half was asserted by reading `verify.build_task`'s source for a substring. Everything `install` controls could therefore be wired to the graded value with the suite green, and that failure is silent in the worst way: the baseline becomes a second graded run, every case comes out `no_signal`, and the report says the skill earns nothing. `inspect_swe` is not installed in the unit job, so `claude_code` is stubbed as `TestEveryLegThinksAsHardAsItWasTold` already stubs it, returning a real solver so `chain` still builds. The staging solver is recorded the same way. Both halves are built, because every assertion here reads the same on one half alone. Nine mutations, each applied alone, each now caught: - the baseline staging the skill anyway, and `skills=[skill_dir]` reaching the agent -- the two that make a baseline not a baseline; - the `-baseline` task name, the `BASELINE_KEY` metadata and the `baseline` tag, which are how the two halves are told apart in the log directory; - `verify.run` dropping `install` on its way to `build_task`, which builds a graded task out of a baseline run, and dropping `epochs`; - `behavioral.run` dropping `baseline=baseline`, the same join on the other engine; - `cmd_staleness` handing the baseline `None` for its config dir, which is what keeps the runner's own `~/.claude/skills` out of the room the whole measurement rests on. Tests only; no production code changed. 809 pass, ruff `F401,F841` clean. --- tests/test_skillscope.py | 116 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 8a49828..d2cf07e 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -7618,6 +7618,26 @@ def test_the_repeat_count_reaches_the_engine(self) -> None: self.assertEqual(seen.get("epochs"), 3) self.assertEqual(seen.get("max_samples"), 8) + def test_which_half_it_is_reaches_the_task(self) -> None: + # `baseline` names the task, stamps `BASELINE_KEY` and tags the log. + # Dropping it from this call builds a graded task out of a baseline + # run, so the two halves land in the log directory under one name + # and the metadata says both were graded. + seen = {} + with mock.patch.object(engine_behavioral, "sandbox_spec"), \ + mock.patch.object(engine_behavioral, "stop_requested", lambda: False), \ + mock.patch.object(engine_behavioral, "build_task", + lambda *a, **k: seen.update(k) or "task"), \ + mock.patch.object(engine_behavioral, "models", mock.MagicMock()), \ + mock.patch.object(engine_behavioral, "run_eval", + lambda ev, task, **kw: []), \ + contextlib.redirect_stdout(io.StringIO()): + engine_behavioral.run( + ["alpha"], self._cases()[:1], "mockllm/model", "high", + solver_factory=None, baseline=True, + ) + self.assertTrue(seen.get("baseline")) + def test_an_ordinary_run_asks_for_nothing_extra(self) -> None: # `behavioral` and `routing` must be untouched by flags only # `staleness` carries. @@ -8073,6 +8093,7 @@ def test_the_other_engine_records_it_too(self) -> None: "SKILL_DIR_KEY", inspect.getsource(engine_verify.build_task) ) + def test_the_verdict_is_the_same_from_any_directory(self) -> None: # The property, asserted as a property. A relative path resolves # against whoever is running, so resolving one here made the answer a @@ -8169,6 +8190,81 @@ def test_a_marker_cannot_reach_the_report_as_an_activation(self) -> None: self.assertEqual(totals["earns_it"], 1) +class TestTheSandboxedBaselineIsGivenNoSkill(unittest.TestCase): + """The baseline half of the `claude-code` leg, built and read. + + `install` was asserted by grepping this function's source, so the whole + of what it controls -- the staging solver, the agent's own `skills` + argument, the task name, the metadata and the tags -- could be wired to + the graded value with the suite green. That failure is silent in the + worst way: the baseline becomes a second graded run, every case comes + out `no_signal`, and the report says the skill earns nothing. + + `inspect_swe` is not installed in the unit job, so `claude_code` is + stubbed the way `TestEveryLegThinksAsHardAsItWasTold` stubs it. The stub + returns a real solver so `chain` still builds. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + # The behavioural dataset, not `tier0_dataset`: `verify.run` skips a + # skill with no case asserting a behaviour, so the graded leg would + # never be built. + self.repo.skill("run", dataset=_behavioral_dataset("run")) + self.repo.activate() + self.cases = [c for c in datasets.load_all_cases() + if c.skill == "run" and c.has_behavior][:1] + + def _task(self, install: bool): + seen: dict = {} + fake = types.ModuleType("inspect_swe") + fake.claude_code = lambda **k: seen.update(agent=k) or engine_verify._ensure_workdir() + staging = lambda skills: seen.update(staged=skills) or engine_verify._ensure_workdir() # noqa: E731 + with mock.patch.dict(sys.modules, {"inspect_swe": fake}), \ + mock.patch.object(engine_verify, "complete_skills_solver", staging): + task = engine_verify.build_task( + "run", self.cases, "mockllm/model", effort="high", install=install, + ) + return task, seen + + def test_the_baseline_stages_nothing_and_is_told_nothing(self) -> None: + task, seen = self._task(install=False) + self.assertEqual(seen["staged"], {}, "the baseline staged the skill") + self.assertEqual(seen["agent"]["skills"], [], "the agent was given the skill") + self.assertTrue(task.metadata[engine_behavioral.BASELINE_KEY]) + self.assertIn("baseline", task.tags) + self.assertTrue(task.name.endswith("-baseline")) + + def test_the_graded_half_gets_the_skill_both_ways(self) -> None: + # The control. Each assertion above reads the same on both halves + # unless the other half is built too. + task, seen = self._task(install=True) + self.assertEqual(list(seen["staged"]), ["run"]) + self.assertEqual(len(seen["agent"]["skills"]), 1) + self.assertFalse(task.metadata[engine_behavioral.BASELINE_KEY]) + self.assertIn("graded", task.tags) + self.assertFalse(task.name.endswith("-baseline")) + + def test_the_runner_hands_its_half_down_to_the_task(self) -> None: + # The join: `run(install=False)` has to reach `build_task`. Dropping + # it there builds a graded task from a baseline run, which no + # assertion above can see. + seen: dict = {} + with mock.patch.object(engine_verify, "require", lambda: None), \ + mock.patch.object(engine_verify.sandbox_spec, "require_provider", + lambda: None), \ + mock.patch.object(engine_verify, "build_task", + lambda *a, **k: seen.update(k) or "task"), \ + mock.patch.object(engine_verify.behavioral, "run_eval", + lambda ev, task, **kw: seen.update(kw) or []): + engine_verify.run( + ["run"], self.cases, "mockllm/model", "high", + install=False, attempts=3, + ) + self.assertFalse(seen["install"]) + self.assertEqual(seen["epochs"], 3) + + class TestContaminationIsJudgedOnTheSkillsRealPath(unittest.TestCase): """A bare skill name is not evidence that the skill's files were read. @@ -8597,6 +8693,26 @@ def test_both_halves_grade_the_same_cases(self) -> None: self.assertEqual(harness.calls[0]["cases"], harness.calls[1]["cases"]) self.assertTrue(harness.calls[0]["cases"]) + def test_the_baseline_half_is_given_a_config_dir_of_its_own(self) -> None: + # The dir the comment calls what the whole command rests on: without + # it the runner's own `~/.claude/skills` are registered beside the + # staged ones and the baseline is this machine's skills minus the one + # under test. The harness recorded it and nothing read it, so handing + # both halves the same dir -- or `None` -- was green. + # The isolation check reads the environment, which another test in + # the suite may have cleared, so it is answered here rather than left + # to whatever ran before this. Its own refusal is covered by + # `test_the_command_refuses_an_unisolatable_host_run`. + with _staleness_harness(self) as harness: + with mock.patch.object(cli.routing, "can_isolate_config", lambda: True): + harness.run(engine="claude-code-no-sandbox") + graded, baseline = harness.calls + self.assertIsNotNone(baseline["config_dir"], "the baseline shares the host's") + self.assertNotEqual( + graded["config_dir"], baseline["config_dir"], + "one dir between the halves is one room between them", + ) + def test_the_two_halves_are_billed_apart_in_the_report(self) -> None: # End to end, through the real `cmd_staleness`. The stub spends a # different amount per half, so a scope that captured both halves From 8bfbc568e1b0be15b4894ae07bc46866819ed104 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Tue, 17 Nov 2026 13:37:02 +0100 Subject: [PATCH 16/17] Pin the report's caveats, and stop citing code that was cut The last two blocking findings. **The caveats the report carries were untested.** Nine branches, each mutated alone and each green: the `--max-cases` banner, the multi-attempt comparability banner, the unknown-room note, the errored-attempts note, `_detail`'s pipe and newline escape, `per_skill`'s keys, `baseline_unknown`, the `--max-cases` half of the negative-count refusal, and the `max_cases` and `concurrency` meta keys. None of them can make the command reach a wrong verdict; all of them are how a reader knows what the verdicts are worth. A capped run that does not say it was capped reads as a statement about the whole suite, and the comparison banner is the only thing in the report saying a three-attempt run is not comparable with a one-attempt one. The `--max-cases` refusal was the one real behaviour among them: only `--concurrency` was covered, so dropping `--max-cases` from the condition let a cap of -1 reach `_gradable_cases`, where a negative slice silently selects nothing. **And comments citing things this round removed.** `_inspect_room`'s `skill_dir` and `_skill_marks`'s `recorded` have no caller in the package: the offline reader they are for went out with `--from-logs`. They are kept rather than deleted, because `SKILL_DIR_KEY` is written to every log from now on and a reader arriving to find the seam missing would have to resolve the skill against itself, which is the one answer always wrong for somebody else's run -- but the docstrings now say that is what they are waiting for, instead of describing a reader that exists. Likewise a comment justifying `epochs` by `staleness._no_duplicates`, which no longer exists, and four test docstrings naming `--from-logs` as a current flag. 816 tests pass, ruff `F401,F841` clean. Every branch above was confirmed by mutation, including one whose first attempt failed to apply and reported a false pass until it was re-run. --- skillscope/engine/behavioral.py | 25 +++++-- tests/test_skillscope.py | 126 ++++++++++++++++++++++++++++---- 2 files changed, 130 insertions(+), 21 deletions(-) diff --git a/skillscope/engine/behavioral.py b/skillscope/engine/behavioral.py index 9411a27..64e0031 100644 --- a/skillscope/engine/behavioral.py +++ b/skillscope/engine/behavioral.py @@ -346,9 +346,11 @@ def _skill_marks(skill: str, recorded: str | None = None) -> tuple[str, ...] | N run. `recorded` is `SKILL_DIR_KEY` off the log: where the skill was on the - machine that ran the agent. Preferred over resolving it here, because - resolving it here resolves it against *this* process. A reader standing - somewhere else is not evidence about where the agent stood. + machine that ran the agent. It has no caller in the package yet, for the + reason `_inspect_room`'s `skill_dir` has none -- the offline reader is + scoped out of this round -- and exists because resolving the path here + resolves it against *this* process. A reader standing somewhere else is + not evidence about where the agent stood. `None` when neither source can say where the skill was, which is an unknown room rather than a clean one. Returning the staged mark alone @@ -483,9 +485,15 @@ def _inspect_room( ) -> tuple[str | None, str | None]: """`(the skill that fired, why the room was not what it claimed)`. - `skill_dir` is `SKILL_DIR_KEY` off the log, passed by a reader rebuilding - someone else's run. A live run leaves it `None` and resolves the skill - where it already knows it is. + `skill_dir` is `SKILL_DIR_KEY` off the log. Nothing in the package passes + it today: the offline reader it is for -- `--from-logs`, which rebuilt a + comparison from logs on disk -- was cut from this round and comes back on + its own. The parameter and the key are kept together deliberately, because + the key is written to every log from now on and a reader that arrives to + find the seam missing would have to resolve the skill against *itself*, + which is the one answer that is always wrong for somebody else's run. A + live run leaves it `None` and resolves the skill where it already knows it + is; the tests cover the other branch until that reader lands. One walk over the tool calls, answering both questions, because the second is only meaningful beside the first. `detect_activation` reports a skill @@ -803,8 +811,9 @@ def run( # Repeats are inspect's epochs: one task, one log, each case # sampled `attempts` times with the epoch on every sample. # Running the leg twice over would have written two logs a - # side, which is the shape `staleness._no_duplicates` refuses - # precisely because it cannot tell it from a collision. + # side, which a reader pairing them cannot tell from two logs + # that collided -- so the repeats stay inside one log and the + # epoch on each sample is what tells them apart. epochs=attempts, **({"max_samples": concurrency} if concurrency else {}), # A sample that raises is that case's failure, not the batch's. diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index d2cf07e..ed150ab 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -6470,8 +6470,8 @@ class TestARepeatIsTakenFromTheSampleThatRanIt(unittest.TestCase): `attempt=int(getattr(sample, "epoch", 1) or 1)` -- could have been hard-coded to 1 or read the wrong attribute with the suite still green. That line is what keeps `--attempts` repeats of one case apart: collapse - it and every repeat collides in `_no_duplicates`, and `_by_case` has one - attempt where the run had three. + it and `_by_case` has one attempt where the run had three, with the + others silently dropped. """ @staticmethod @@ -6498,7 +6498,7 @@ def test_each_repeat_carries_its_own_number(self) -> None: def test_a_sample_with_no_epoch_is_the_first_attempt(self) -> None: # Ordinary single-pass runs, and any log written before epochs were # asked for, have to read as attempt 1 rather than 0 -- `_by_case` - # and `_no_duplicates` both key on it. + # keys on it. sample = _StubSample("c1", self._checks()) del sample.epoch log = types.SimpleNamespace(status="success", error=None, samples=[sample]) @@ -7478,6 +7478,87 @@ def test_one_attempt_a_side_can_never_read_as_unsettled(self) -> None: self.assertIn("already did this unaided", report) +class TestTheCaveatsTheReportCarries(unittest.TestCase): + """Report branches that each survived being turned off. + + All of them are prose or a key in the JSON rather than a verdict, so + none of them can make the command reach a wrong answer. What they can do + is let a reader take a partial run for a whole one, or compare two runs + that are not comparable, which is the one way this report misleads + without being wrong. + """ + + @staticmethod + def _summary(comparisons, **meta): + return staleness.summarize(comparisons, meta) + + def _report(self, comparisons, **meta): + return staleness.render_markdown(self._summary(comparisons, **meta)) + + @staticmethod + def _pair(**kwargs): + return staleness.pair( + [_outcome("c", True, **kwargs)], [_outcome("c", False, **kwargs)] + ) + + def test_a_capped_run_says_it_covered_only_part_of_the_suite(self) -> None: + self.assertIn("covered only the first 2 case(s)", + self._report(self._pair(), max_cases=2)) + self.assertNotIn("covered only the first", self._report(self._pair())) + + def test_a_multi_attempt_run_says_what_it_compares_with(self) -> None: + # The verdict's meaning moves with the attempt count, and nothing + # else in the report says so. + many = staleness.pair( + [_outcome("c", True, attempt=1), _outcome("c", True, attempt=2)], + [_outcome("c", False, attempt=1), _outcome("c", False, attempt=2)], + ) + self.assertIn("not with runs at a different count", self._report(many)) + self.assertNotIn("not with runs at a different count", + self._report(self._pair())) + + def test_a_room_nobody_could_check_is_named_as_such(self) -> None: + comparisons = self._pair() + comparisons[0].baseline_contaminated = engine_behavioral.UNKNOWN_ROOM + self.assertIn("could not be checked for a clean baseline", + self._report(comparisons)) + self.assertEqual( + self._summary(comparisons)["totals"]["baseline_unknown"], 1 + ) + + def test_errored_attempts_are_said_out_loud(self) -> None: + # A reader comparing `2/2` against `3/3` on the next case cannot + # otherwise tell the first was asked three times. + comparisons = staleness.pair( + [_outcome("c", True, attempt=1), + _outcome("c", False, attempt=2, error="boom")], + [_outcome("c", False, attempt=1)], + ) + self.assertIn("attempt(s) errored and are out of", self._report(comparisons)) + self.assertNotIn("errored and are out of", self._report(self._pair())) + + def test_a_detail_cannot_break_the_table_it_sits_in(self) -> None: + # A pipe closes the cell and a newline closes the row, so either one + # turns the rest of the report into something the reader cannot + # parse. Both come from a judge's own words. + self.assertEqual( + staleness._detail( + {"passed": False, "detail": "it ran `a | b`\nthen stopped"} + ), + "it ran `a \\| b` then stopped", + ) + + def test_the_summary_counts_each_skill_separately(self) -> None: + comparisons = staleness.pair( + [_outcome("a", True, skill="one"), _outcome("b", True, skill="two")], + [_outcome("a", False, skill="one"), _outcome("b", True, skill="two")], + ) + per_skill = self._summary(comparisons)["per_skill"] + self.assertEqual(sorted(per_skill), ["one", "two"]) + self.assertEqual(per_skill["one"]["earns_it"], 1) + self.assertEqual(per_skill["two"]["earns_it"], 0) + + class TestAReportWithNothingGradedAssertsNothingAboutGradedCases(unittest.TestCase): """Sentences whose condition is not the thing they assert. @@ -7660,10 +7741,8 @@ def test_an_ordinary_run_asks_for_nothing_extra(self) -> None: class TestAFlagThatCannotDoAnythingIsRefused(unittest.TestCase): """The repo's habit: refuse an argument rather than accept and ignore it. - A `--from-logs` rebuild starts no agent. The attempts, the cap and the - concurrency are properties of the run that wrote the logs, so accepting - them would let somebody believe they had re-measured at three attempts - when they had reread one. + A flag the command cannot act on is refused rather than accepted and + ignored, so nobody believes they measured something they did not. """ def _run(self, *argv): @@ -7676,7 +7755,11 @@ def test_a_run_of_no_attempts_is_refused(self) -> None: self.assertIn("would run nothing", self._run("--attempts", "0")) def test_a_negative_count_is_refused(self) -> None: + # Both halves of the condition. Only `--concurrency` was covered, so + # dropping `--max-cases` from it left a cap of -1 to reach + # `_gradable_cases`, where a negative slice silently selects nothing. self.assertIn("cannot be negative", self._run("--concurrency", "-1")) + self.assertIn("cannot be negative", self._run("--max-cases", "-1")) def test_a_harness_with_no_staleness_leg_is_refused(self) -> None: # The baseline half is only a measurement if the room is empty, and @@ -7694,10 +7777,10 @@ def test_the_default_harness_is_not_refused(self) -> None: # The refusal is about the room check, not about the flag. Naming the # default explicitly has to behave exactly like not naming it. # - # This used to pass `--from-logs`, which returned before the guard - # was ever reached -- so it asserted the absence of a refusal that - # could not have fired, and would have gone on passing had the guard - # been written to refuse the default too. + # This used to pass a flag that returned before the guard was ever + # reached -- so it asserted the absence of a refusal that could not + # have fired, and would have gone on passing had the guard been + # written to refuse the default too. # # Stopped at the preflight rather than let through it. Past the # guard the next thing `cmd_staleness` does is `_prepare_graded_run`, @@ -8413,8 +8496,9 @@ class TestAWalkThatDidNotFinishCannotVouchForTheRoom(unittest.TestCase): `_activation_event` reaches `call.function` with no `getattr` guard, unlike every other accessor on this path, so an unexpected call shape is - the realistic way in -- and `--from-logs`, reading logs this code did not - write, is exactly where an unexpected shape would come from. + the realistic way in -- and the offline reader this round scoped out, + which will read logs this code did not write, is exactly where an + unexpected shape would come from. """ def setUp(self) -> None: @@ -8786,6 +8870,22 @@ def test_an_unhurried_run_marks_both_halves_finished(self) -> None: harness.run() self.assertTrue(all(half["finished"] for half in harness.reports[0]["meta"]["legs"])) + def test_the_caps_that_bound_a_run_are_recorded_in_its_report(self) -> None: + # The report's own caveat about a partial run is driven off these + # keys, so dropping them silently turns a capped run into one that + # reads as whole. Recorded only when they bound: a `0` would read as + # a cap of nothing rather than as no cap. + with _staleness_harness(self) as harness: + harness.run(extra=["--max-cases", "2", "--concurrency", "4"]) + meta = harness.reports[0]["meta"] + self.assertEqual(meta["max_cases"], 2) + self.assertEqual(meta["concurrency"], 4) + + with _staleness_harness(self) as plain: + plain.run() + self.assertNotIn("max_cases", plain.reports[0]["meta"]) + self.assertNotIn("concurrency", plain.reports[0]["meta"]) + def test_the_report_names_the_harness_like_every_other_one(self) -> None: # `cmd_staleness` hand-wrote its meta, so a staleness report was the # one graded report with no `harness` or `harness_version` in it, and From 723429d85ebf8cd9dc3ab438dee5eaef53b1a789 Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Wed, 18 Nov 2026 15:09:17 +0100 Subject: [PATCH 17/17] Catch a sweep split across arguments, and a sample the clock stopped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four blockers from the fourth review pass. **A sweep through Glob or Grep read as a clean room.** Both the sweep test and the read test ran per string argument, so a sweep needed its word and a skill-ancestor directory in the same string. A Bash command is one string and was caught; Glob and Grep carry the two halves as separate arguments, and those are the tools an agent that has been told nothing reaches for first. Probed on the last head: `{"pattern": "**/SKILL.md", "path": }` came back `None` while the identical sweep typed as a shell command was flagged. The sweep question is now asked of the whole call rather than of each argument. And a search tool needs no sweep word at all: `Grep(pattern= "deploy", path=".claude")` reads every file under `.claude` and carries no wildcard, no `-r` and no path naming the skill, so the only evidence is which tool was called. `_searches` says so by function name, loosely, because the same two tools are spelled differently by different harnesses. **A sample inspect cut short was graded as an ordinary failure.** Nothing read `sample.limit`, which inspect sets separately from `sample.error`, so an agent the clock stopped arrived as `passed=False, error=None` and `answered` kept it. The baseline runs second on whatever budget the graded half left and is the half that explores longest, so it is the one the limits cut -- and a baseline interrupted rather than beaten reads as a case the skill earned. Time, working-time, message and token limits now error the attempt, which lands the case `ungraded`. `operator` and `custom` are left alone: one is somebody cancelling and the other is a task's own rule. **The write-only keys and their parameters are gone.** `SKILL_DIR_KEY`, `SKILL_KEY` and `BASELINE_KEY` were written to every log by both engines and read nowhere, and `_inspect_room(skill_dir=…)` and `_skill_marks(recorded=…)` had no caller. Keeping them and correcting the comments was the wrong answer last round: the `usage.md` paragraph went further than the comments did and described a reader that does not exist. Deleted, with the tests that only covered them and that paragraph. The tags and the task name already tell the two halves apart in a log directory. They come back with the reader. **And the tests that could not fail.** The sharpest: `_staleness_harness` returned no outcomes from either half, so no verdict was ever compared and `pair(with_skill, without)` could be called with its arguments swapped with the suite green -- every real report would have had `earns_it` and `harmful` the wrong way round. The halves now return outcomes that differ, and the orientation is asserted on both the verdict and the time column. Also: a third vacuous `assertNotIn` naming a string the code never emits; `_outcomes` calling `_inspect_room` at all; `usage.leg`'s `finally`; and six report branches with no assertion. Two of those needed a second attempt. The `usage.leg` test first asserted a total, which comes out the same either way because the fold-back adds exactly what the leaked accumulator already holds; it asserts identity now. And the first mutation of that `finally` was a syntax error rather than a test, so its pass meant nothing until it was re-run properly. 829 tests pass, ruff `F401,F841` clean. Every fix above confirmed by mutation. --- docs/usage.md | 11 - skillscope/engine/behavioral.py | 175 ++++++++------ skillscope/engine/verify.py | 8 - tests/test_skillscope.py | 413 ++++++++++++++++++++++---------- 4 files changed, 395 insertions(+), 212 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 103973e..7b08688 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -360,17 +360,6 @@ without it explores. Three flags decide what a run costs. | `--max-cases N` | every case | Grade only the first N cases of each selected skill, after `--only` and `--extended` have chosen them. Both sides run the same N, and the report says the run did not cover the whole suite. | | `--concurrency N` | inspect's default | How many cases may be in flight at once. Where the wall clock goes on the baseline half. Raise it as far as the engine's sandboxes and the model's rate limits allow. | -Both halves write which side they were, whose skill they were about, and where -that skill was on disk into their inspect log metadata (`skillscope_baseline`, -`skillscope_skill`, `skillscope_skill_dir`). - -`skillscope_skill_dir` is the load-bearing one. It is what lets the baseline be -judged empty: the check recognises a tool call that reached the skill by -comparing against the skill's absolute path, so a reader working that path out -for itself would be working out its own rather than the run's, and would report -a baseline the run had voided as a clean one. A log written without the key is -read as an unknown room rather than an empty one. - Budget more wall clock than a `behavioral` run of the same cases, and more than double it. `--timeout` defaults to twice what the single-pass commands use for that reason. Both passes share one `--timeout`, and the baseline is diff --git a/skillscope/engine/behavioral.py b/skillscope/engine/behavioral.py index 64e0031..3cd5e16 100644 --- a/skillscope/engine/behavioral.py +++ b/skillscope/engine/behavioral.py @@ -118,12 +118,6 @@ def build_task( bound = deadline.active() return Task( name=f"behavioral-{skill}" + ("-baseline" if baseline else ""), - metadata={ - BASELINE_KEY: baseline, - SKILL_KEY: skill, - # Resolved here, where it is known to be right. See `SKILL_DIR_KEY`. - SKILL_DIR_KEY: str(Path(skill_dir).resolve()), - }, tags=["skillscope", "baseline" if baseline else "graded"], dataset=samples, setup=hooks.setup_solver(hook, skill, cases), @@ -252,6 +246,24 @@ def _expand_home(text: str) -> str: ) +# Tools whose whole job is to walk a tree. A call to one of these is a sweep +# whatever its arguments say, because the recursion is the tool rather than +# something the agent spelled out: `Grep(pattern="deploy", path=".claude")` +# reads every file under `.claude` and carries no wildcard, no `-r` and no +# path that names the skill. +# +# Matched on the function name, loosely, because the same two tools are +# spelled differently by different harnesses and an MCP server may wrap them +# under a name of its own. +_SEARCH_TOOLS = ("glob", "grep", "search") + + +def _searches(call) -> bool: + """Whether this call is a directory search by virtue of which tool it is.""" + name = _normalise(str(getattr(call, "function", "") or "")) + return any(tool in name for tool in _SEARCH_TOOLS) + + def _swept(text: str, marks: tuple[str, ...]) -> bool: """Whether this call read a directory containing the skill, wholesale. @@ -328,7 +340,7 @@ def _ancestors(marks: tuple[str, ...]) -> set[str]: ) -def _skill_marks(skill: str, recorded: str | None = None) -> tuple[str, ...] | None: +def _skill_marks(skill: str) -> tuple[str, ...] | None: """The paths that reading makes a room dirty, lowercased and separator-flat. Two of them. The skill's own directory, absolute, which is where an agent @@ -339,24 +351,23 @@ def _skill_marks(skill: str, recorded: str | None = None) -> tuple[str, ...] | N so this asks it rather than spelling one harness's answer in, which would read every other harness's staged copy as untouched. - It asks the *default* harness, which is right while that is the only one - registered. Once a run can name another, the staged mark has to come off - the log beside `recorded`, for the reason `recorded` exists: a reader - standing on a different harness than the run is not evidence about the - run. - - `recorded` is `SKILL_DIR_KEY` off the log: where the skill was on the - machine that ran the agent. It has no caller in the package yet, for the - reason `_inspect_room`'s `skill_dir` has none -- the offline reader is - scoped out of this round -- and exists because resolving the path here - resolves it against *this* process. A reader standing somewhere else is - not evidence about where the agent stood. - - `None` when neither source can say where the skill was, which is an - unknown room rather than a clean one. Returning the staged mark alone - looked like an answer and was not: it is the subset of the check that - cannot fail, so every tool call that reached the skill by its real path - came back clean and the case was credited. See `SKILL_DIR_KEY`. + It asks the *default* harness, which is right while that is the only + harness this command will run on -- `staleness` refuses any other, and + `behavioral` on another harness stages its skill somewhere this would not + recognise. Worth revisiting the day that refusal lifts. + + The absolute mark is resolved here, against this process. That is right + for every caller there is: the check runs in the same process as the run + it is judging. It would be wrong for a reader rebuilding somebody else's + logs, which is why that reader will have to carry the path in the log + rather than work it out -- but no such reader exists yet, and a parameter + kept warm for one is a parameter nothing tests against reality. + + `None` when it cannot say where the skill was, which is an unknown room + rather than a clean one. Returning the staged mark alone looked like an + answer and was not: it is the subset of the check that cannot fail, so + every tool call that reached the skill by its real path came back clean + and the case was credited. The staged mark carries no leading separator on purpose. `no_sandbox` runs the agent with `cwd=workspace`, which makes `cat .claude/skills/foo/...` @@ -364,15 +375,13 @@ def _skill_marks(skill: str, recorded: str | None = None) -> tuple[str, ...] | N exactly that. `_mentions` supplies the boundary instead, which is what keeps the mark from matching the middle of a longer name. """ - home = recorded - if home is None: - try: - home = str(Path(config.active().skill_path(skill)).resolve()) - except (Exception, SystemExit): # noqa: BLE001 -- see below - # `skill_path` raises `SystemExit` rather than returning, so - # catching `Exception` alone let a rebuild of someone else's logs - # take the whole command down. - return None + try: + home = str(Path(config.active().skill_path(skill)).resolve()) + except (Exception, SystemExit): # noqa: BLE001 -- see below + # `skill_path` raises `SystemExit` rather than returning, so catching + # `Exception` alone let a skill the config cannot place take the + # whole command down. + return None staged = f"{harnesses.get().skills_dir}/{skill}" return (_normalise(home), _normalise(staged)) @@ -475,26 +484,37 @@ def _read_the_skill(call, marks: tuple[str, ...], skill: str) -> str | None: # Last, and weaker on purpose: a call that read a directory the skill # sits in, wholesale. Answered after the exact matches above so a call # that provably named the skill keeps the reason that says so. - if any(_swept(text, marks) for text in spellings): + # + # Asked of the call, not of each argument on its own. A Bash sweep is one + # string holding both halves -- the sweep word and the directory -- but + # Glob and Grep take them as separate arguments, `pattern` and `path`, + # and those are the tools an agent that has been told nothing reaches for + # first. Tested per argument, `{"pattern": "**/SKILL.md", "path": + # ""}` had a sweep word in one string and an ancestor in + # another and matched in neither, so the same sweep that was caught as a + # shell command came back as a clean room through the structured tool. + # A call to a search tool needs no sweep word at all: the recursion is + # the tool. `Grep` with `path` set to a directory walks everything under + # it, and `{"pattern": "deploy", "path": ".claude"}` carries no marker + # any string test could find -- the only evidence that this read the + # whole tree is which tool was called. + swept = _searches(call) or any( + glob in text for text in spellings for glob in _SWEEP_GLOBS + ) or any(_SWEEP_WORDS.search(text) for text in spellings) + if swept and any( + _mentions(text, ancestor) + for text in spellings + for ancestor in _ancestors(marks) + ): return UNKNOWN_ROOM return None def _inspect_room( - routing_engine, sample, skill: str, skill_dir: str | None = None + routing_engine, sample, skill: str ) -> tuple[str | None, str | None]: """`(the skill that fired, why the room was not what it claimed)`. - `skill_dir` is `SKILL_DIR_KEY` off the log. Nothing in the package passes - it today: the offline reader it is for -- `--from-logs`, which rebuilt a - comparison from logs on disk -- was cut from this round and comes back on - its own. The parameter and the key are kept together deliberately, because - the key is written to every log from now on and a reader that arrives to - find the seam missing would have to resolve the skill against *itself*, - which is the one answer that is always wrong for somebody else's run. A - live run leaves it `None` and resolves the skill where it already knows it - is; the tests cover the other branch until that reader lands. - One walk over the tool calls, answering both questions, because the second is only meaningful beside the first. `detect_activation` reports a skill nobody staged as `other:`, which is exactly the contamination this @@ -521,7 +541,7 @@ def _inspect_room( """ fired: str | None = None why: str | None = None - marks = _skill_marks(skill, skill_dir) + marks = _skill_marks(skill) # Nowhere to check against, so the read half of the check cannot run. Said # out loud rather than passed over: this lands the case `ungraded`, which # is what an unanswerable question is worth. Applied after the walk, not @@ -554,6 +574,35 @@ def _inspect_room( return fired, why or (UNKNOWN_ROOM if unknown else None) +# Limits that stop an agent mid-answer rather than deciding anything about +# it. `operator` and `custom` are left out deliberately: one is somebody +# cancelling and the other is a task's own rule, and neither is the clock. +_CUT_SHORT = { + "time": "stopped at the sample's time limit", + "working": "stopped at the sample's working-time limit", + "message": "stopped at the message limit", + "token": "stopped at the token limit", +} + + +def _cut_short(sample) -> str | None: + """Why this sample was stopped before it finished, if it was. + + inspect records a limit as `sample.limit`, separately from `sample.error`, + so a run the clock ended arrives looking exactly like one the agent + finished and failed. Read defensively: `limit` is absent on older logs and + its shape is inspect's rather than ours. + """ + limit = getattr(sample, "limit", None) + if limit is None: + return None + kind = str(getattr(limit, "type", None) or limit).lower() + for name, reason in _CUT_SHORT.items(): + if name in kind: + return reason + return None + + def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: """Map one inspect `EvalLog` back onto skillscope's outcome objects. @@ -590,6 +639,16 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: if sample.error is not None: error = f"{sample.error.message}" + elif (cut := _cut_short(sample)) is not None: + # An agent the clock stopped did not answer the question; it ran + # out of room to answer it. inspect reports that as `sample.limit` + # and leaves `error` unset, so the attempt arrived as an ordinary + # failure -- which on the baseline is the dangerous direction. The + # baseline runs second, on whatever budget the graded half left, + # and is the half that explores longest, so it is the one the + # limits cut. A case the skill never earned then reads `earns_it`, + # off a baseline that was interrupted rather than beaten. + error = cut elif not checks: error = "case has no behavioral assertions to grade" elif ungraded := [c for c in checks if not scorers.graded(c)]: @@ -647,28 +706,6 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: CALL_TOKEN = "skillscope_call" # Which half of a staleness comparison a log is, and whose skill it is about. -# The task name says the same thing, but a name is a string somebody has to -# parse; inspect filters on metadata and tags, and so can anything reading a -# log directory afterwards. Written by both engines, so a reader does not have -# to know which one produced the log it is holding. -BASELINE_KEY = "skillscope_baseline" -SKILL_KEY = "skillscope_skill" - -# Where the skill was on the machine that ran the agent. -# -# `_skill_marks` needs an absolute directory to recognise a tool call that -# reached into the skill. Deriving it at read time derives it from whoever is -# *reading*: a rebuild in another directory, or on a machine where the skill -# is not checked out, built a quietly weaker mark set and reported a baseline -# the live run had voided as `ungraded` back as `no_signal`. That is the -# "a rebuild is more permissive than the run it rebuilds" failure this module -# closes everywhere else; contamination was the one place it was still open. -# -# So the run records where the skill actually was, and the reader uses that -# rather than its own cwd. Written by both engines for the same reason -# `SKILL_KEY` is: a reader should not have to know which one produced the log. -SKILL_DIR_KEY = "skillscope_skill_dir" - def stop_requested() -> bool: """Whether the deadline's graceful stage has asked the command to stop.""" diff --git a/skillscope/engine/verify.py b/skillscope/engine/verify.py index 9e99652..5d81935 100644 --- a/skillscope/engine/verify.py +++ b/skillscope/engine/verify.py @@ -249,14 +249,6 @@ def build_task( bound = deadline.active() return Task( name=f"claude-code-{skill}" + ("" if install else "-baseline"), - # See `behavioral.build_task`: the same keys whichever engine ran. - metadata={ - behavioral.BASELINE_KEY: not install, - behavioral.SKILL_KEY: skill, - # See `behavioral.SKILL_DIR_KEY`: resolved on the machine that ran - # the agent, because a reader cannot work it out afterwards. - behavioral.SKILL_DIR_KEY: str(Path(skill_dir).resolve()), - }, tags=["skillscope", "graded" if install else "baseline"], setup=hooks.setup_solver(hook, skill, cases), cleanup=hooks.cleanup_fn(hook, skill, cases), diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index ed150ab..7a75f50 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -6330,7 +6330,22 @@ def record(args, skills, gradable, *, install=True, config_dir=None): output_tokens=1 if install else 20, calls=1 if install else 7, ) - return [] + # Outcomes that differ by half, so the verdict the command computes + # is one only the right pairing can produce. Returning `[]` from both + # halves -- which this did -- meant no verdict was ever compared, so + # `pair(with_skill, without)` could be called with its arguments + # swapped and every test stayed green, while every real report had + # `earns_it` and `harmful` the wrong way round. + return [ + behavior.BehaviorOutcome( + id=case.id, skill=case.skill, prompt=case.prompt, + passed=install, elapsed_s=1.0 if install else 9.0, + checks=[{"kind": "expected_behavior", "expectation": "e", + "passed": install, "detail": ""}], + activated="alpha" if install else None, + ) + for case in gradable + ] reports: list[dict] = [] harness = types.SimpleNamespace(repo=repo, calls=calls, reports=reports) @@ -6437,7 +6452,10 @@ def test_a_clean_run_is_not_described_as_every_case_needing_the_skill(self) -> N summary = staleness.summarize(comparisons, {"model": "m", "effort": "e"}) report = staleness.render_markdown(summary) self.assertEqual(summary["totals"]["no_signal"], 1) - self.assertNotIn("every case needed the skill", report) + # The string the code emits is "Every graded case needed the skill"; + # the lower-cased paraphrase this used to assert the absence of + # appears nowhere, so that line could not fail. + self.assertNotIn("Every graded case needed the skill", report) self.assertIn("every claim came out the same", report) def test_a_check_only_one_side_graded_is_still_shown(self) -> None: @@ -6623,9 +6641,7 @@ def _why(self, arguments, function="Bash"): call = _FakeCall("t", function=function) call.arguments = arguments sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) - return engine_behavioral._inspect_room( - routing_engine, sample, "run", self.own - )[1] + return engine_behavioral._inspect_room(routing_engine, sample, "run")[1] def test_a_parent_traversal_names_the_skill_and_is_contamination(self) -> None: # Provable: the path resolves to the skill with no cwd needed. @@ -6790,6 +6806,157 @@ def test_the_sandboxed_leg_carries_the_same_three(self) -> None: self.assertEqual(seen["concurrency"], 4) +class TestTheRoomCheckIsWiredIntoTheOutcome(unittest.TestCase): + """`_outcomes` calling `_inspect_room` was itself untested. + + Every test drove `_inspect_room` directly or hand-built a + `BehaviorOutcome`, so the call could be replaced with `None, None`, or + `activated=`/`contaminated=` dropped from the constructor, and the whole + suite stayed green -- with the room check still passing its own tests + while nothing downstream ever saw its answer. + """ + + @staticmethod + def _case(case_id: str): + return datasets.Case( + id=case_id, skill="alpha", prompt="p", skill_should_trigger=True, + ) + + def test_what_the_room_check_found_reaches_the_outcome(self) -> None: + sample = types.SimpleNamespace( + id="c1", + scores={"s": types.SimpleNamespace(metadata={engine_scorers.CHECKS: [ + {"kind": "expected_behavior", "expectation": "e", + "passed": True, "detail": "", "graded": True}, + ]})}, + error=None, limit=None, total_time=1.0, + ) + log = types.SimpleNamespace(status="success", error=None, samples=[sample]) + with mock.patch.object( + engine_behavioral, "_inspect_room", + lambda *a, **k: ("alpha", "another skill fired: beta"), + ): + outcome = engine_behavioral._outcomes(log, "alpha", [self._case("c1")])[0] + self.assertEqual(outcome.activated, "alpha") + self.assertEqual(outcome.contaminated, "another skill fired: beta") + + +class TestASweepSplitAcrossArgumentsIsStillASweep(unittest.TestCase): + """Glob and Grep carry the sweep and the directory as separate arguments. + + Every test was testing each string argument on its own, so a sweep needed + its word and an ancestor directory in one string. A Bash command is one + string and was caught; `Glob(pattern="**/SKILL.md", path=)` put + the two halves in different arguments and matched in neither, so the same + sweep came back a clean room through the structured tool. Those are the + tools an agent that has been told nothing reaches for first, on the side + whose emptiness the verdict rests on. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill("run", dataset=tier0_dataset("run")) + self.repo.activate() + self.own = Path(config.active().skill_path("run")) + + def _why(self, function, arguments): + from skillscope.engine import routing as routing_engine + + call = _FakeCall("t", function=function) + call.arguments = arguments + sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) + return engine_behavioral._inspect_room(routing_engine, sample, "run")[1] + + def test_a_glob_over_an_ancestor_is_an_unknown_room(self) -> None: + self.assertEqual( + self._why("Glob", {"pattern": "**/SKILL.md", + "path": str(self.own.parent)}), + engine_behavioral.UNKNOWN_ROOM, + ) + + def test_a_grep_needs_no_sweep_word_at_all(self) -> None: + # The half no string test could reach: neither `deploy` nor the + # directory carries a wildcard, an `-r` or the skill's name. That the + # whole tree was read is a fact about which tool was called. + self.assertEqual( + self._why("Grep", {"pattern": "deploy", "path": str(self.own.parent)}), + engine_behavioral.UNKNOWN_ROOM, + ) + + def test_a_search_somewhere_else_is_still_clean(self) -> None: + # The false positive this must not buy: being a search tool is not + # enough on its own, the path still has to reach the skill. + self.assertIsNone(self._why("Grep", {"pattern": "deploy", "path": "/etc"})) + self.assertIsNone( + self._why("Glob", {"pattern": "**/*.py", "path": "build/out"}) + ) + + def test_a_bash_sweep_is_unaffected(self) -> None: + self.assertEqual( + self._why("Bash", {"command": f"grep -ri foo {self.own.parent}/"}), + engine_behavioral.UNKNOWN_ROOM, + ) + + +class TestAnAgentTheClockStoppedDidNotAnswerTheQuestion(unittest.TestCase): + """A sample inspect cut short is not a case the agent failed. + + inspect records a limit on `sample.limit` and leaves `sample.error` + unset, so the attempt arrived as an ordinary failure. The baseline runs + second on whatever budget the graded half left and is the half that + explores longest, so it is the one the limits cut -- and a baseline that + was interrupted rather than beaten reads as a case the skill earned. + """ + + @staticmethod + def _case(case_id: str): + return datasets.Case( + id=case_id, skill="alpha", prompt="p", skill_should_trigger=True, + ) + + @staticmethod + def _log(limit): + sample = types.SimpleNamespace( + id="c1", + scores={"s": types.SimpleNamespace(metadata={engine_scorers.CHECKS: [ + {"kind": "expected_behavior", "expectation": "e", + "passed": False, "detail": "did not", "graded": True}, + ]})}, + error=None, + limit=limit, + total_time=1.0, + ) + return types.SimpleNamespace(status="success", error=None, samples=[sample]) + + def _attempt(self, limit): + return engine_behavioral._outcomes( + self._log(limit), "alpha", [self._case("c1")] + )[0] + + def test_a_sample_the_clock_stopped_is_ungraded_not_failed(self) -> None: + outcome = self._attempt(types.SimpleNamespace(type="time")) + self.assertIn("time limit", outcome.error or "") + self.assertEqual(staleness.answered([outcome]), [], + "an interrupted attempt is evidence about nothing") + + def test_the_message_limit_counts_too(self) -> None: + outcome = self._attempt(types.SimpleNamespace(type="message")) + self.assertIn("message limit", outcome.error or "") + + def test_a_sample_that_simply_failed_is_still_a_failure(self) -> None: + # The other direction, and the one a careless fix breaks: a case the + # agent genuinely failed must stay a failure rather than become an + # error the comparison then declines to count. + outcome = self._attempt(None) + self.assertIsNone(outcome.error) + self.assertFalse(outcome.passed) + + def test_a_limit_shape_this_does_not_know_is_not_invented(self) -> None: + # `operator` is somebody cancelling, not the clock. Read defensively + # because the shape is inspect's rather than ours. + self.assertIsNone(self._attempt(types.SimpleNamespace(type="operator")).error) + + class TestTheRecursiveShapesAnAgentActuallyTypes(unittest.TestCase): """The sweep detector matched the tidiest spelling of each shape only. @@ -6819,9 +6986,7 @@ def _why(self, command): call = _FakeCall("t", function="Bash") call.arguments = {"command": command} sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) - return engine_behavioral._inspect_room( - routing_engine, sample, "run", str(self.own) - )[1] + return engine_behavioral._inspect_room(routing_engine, sample, "run")[1] def test_a_recursive_flag_in_any_cluster_is_a_sweep(self) -> None: above = self.own.parent @@ -7061,10 +7226,11 @@ def _walk(self, hits): with mock.patch.object( engine_behavioral.routing_core, "detect_activation", lambda *a, **k: answers.pop(0), + ), mock.patch.object( + engine_behavioral, "_skill_marks", + lambda name: (f"/nowhere/skills/{name}", f".claude/skills/{name}"), ): - return engine_behavioral._inspect_room( - routing_engine, sample, "run", "/nowhere/skills/run" - ) + return engine_behavioral._inspect_room(routing_engine, sample, "run") def test_a_later_contamination_does_not_replace_the_first_reason(self) -> None: fired, why = self._walk(["other:alpha", "other:beta"]) @@ -7548,6 +7714,47 @@ def test_a_detail_cannot_break_the_table_it_sits_in(self) -> None: "it ran `a \\| b` then stopped", ) + def test_a_run_that_graded_nothing_says_so(self) -> None: + # Every case ungraded. The verdict table is all zeroes and true, and + # the banner is the only thing that tells a reader why. + errored = staleness.pair( + [_outcome("c", False, error="boom")], [_outcome("c", False, error="boom")] + ) + report = self._report(errored) + self.assertEqual(self._summary(errored)["totals"]["graded"], 0) + self.assertIn("Nothing was graded both ways", report) + self.assertIn("No case was graded both ways", report) + self.assertNotIn("graded cases needed the skill", report) + + def test_a_harmful_case_is_one_worth_a_look(self) -> None: + # The strongest negative finding the command makes. A `harmful` case + # missing from the table is the one omission a reader cannot + # recover from the counts. + harmful = staleness.pair([_outcome("c", False)], [_outcome("c", True)]) + self.assertEqual(harmful[0].verdict, "harmful") + report = self._report(harmful) + self.assertIn("Cases worth a look", report) + self.assertIn("`c`", report) + + def test_a_case_only_the_baseline_ran_is_still_reported(self) -> None: + # Dropping it would shrink the denominator instead, so a case the + # graded half never reached would quietly improve the headline. + only_baseline = staleness.pair([], [_outcome("c", False)]) + self.assertEqual(len(only_baseline), 1) + self.assertEqual(only_baseline[0].verdict, "ungraded") + self.assertIn("`c`", self._report(only_baseline)) + + def test_an_unknown_room_is_not_counted_as_a_dirty_one(self) -> None: + # Two different findings: one says the baseline demonstrably held + # something, the other that nobody could tell. Folding them together + # would report a room nobody checked as one that was checked and + # found dirty. + comparisons = self._pair() + comparisons[0].baseline_contaminated = engine_behavioral.UNKNOWN_ROOM + totals = self._summary(comparisons)["totals"] + self.assertEqual(totals["baseline_contaminated"], 0) + self.assertEqual(totals["baseline_unknown"], 1) + def test_the_summary_counts_each_skill_separately(self) -> None: comparisons = staleness.pair( [_outcome("a", True, skill="one"), _outcome("b", True, skill="two")], @@ -7700,10 +7907,10 @@ def test_the_repeat_count_reaches_the_engine(self) -> None: self.assertEqual(seen.get("max_samples"), 8) def test_which_half_it_is_reaches_the_task(self) -> None: - # `baseline` names the task, stamps `BASELINE_KEY` and tags the log. - # Dropping it from this call builds a graded task out of a baseline - # run, so the two halves land in the log directory under one name - # and the metadata says both were graded. + # `baseline` names the task and tags the log. Dropping it from this + # call builds a graded task out of a baseline run, so the two halves + # land in the log directory under one name and both are tagged as + # the graded one. seen = {} with mock.patch.object(engine_behavioral, "sandbox_spec"), \ mock.patch.object(engine_behavioral, "stop_requested", lambda: False), \ @@ -8115,89 +8322,8 @@ def test_the_task_name_says_which_half_it_is(self) -> None: ) self.assertFalse(graded.name.endswith("-baseline")) self.assertTrue(baseline.name.endswith("-baseline")) - self.assertFalse(graded.metadata[engine_behavioral.BASELINE_KEY]) - self.assertTrue(baseline.metadata[engine_behavioral.BASELINE_KEY]) - - -class TestTheRoomIsJudgedWhereTheRunWasNotWhereTheReaderIs(unittest.TestCase): - """A rebuilt contamination verdict must not depend on who is reading. - - `_skill_marks` recognises a tool call that reached the skill by comparing - against the skill's absolute path. Worked out at read time it is the - *reader's* answer: rebuilt from another directory, or on a machine without - the skill checked out, the mark went missing and a baseline the run had - voided as `ungraded` came back `no_signal` -- stated in the report as - fact. So the run records where the skill was and the reader uses that. - - This is the same hazard `test_a_rebuild_is_no_more_permissive_than_the_run_ - it_rebuilds` guards for the no-checks branch, in the one place it was - still open. - """ - - def setUp(self) -> None: - self.repo = Repo(self) - self.repo.skill("run", dataset=tier0_dataset("run")) - self.repo.activate() - self.cases = [c for c in datasets.load_all_cases() if c.skill == "run"][:1] - - def _why(self, arguments, skill_dir=None): - from skillscope.engine import routing as routing_engine - - call = _FakeCall("toolu_1", function="Read") - call.arguments = arguments - sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) - return engine_behavioral._inspect_room( - routing_engine, sample, "run", skill_dir - )[1] - - def test_the_run_records_where_the_skill_was(self) -> None: - # Without this the reader has only its own cwd to go on, and the - # rebuild silently returns to deriving the weaker answer -- which on - # the machine that wrote the logs looks exactly right. - own = str(config.active().skill_path("run").resolve()) - for baseline in (False, True): - task = engine_behavioral.build_task( - "run", self.cases, "mockllm/model", - solver_factory=lambda d: None, baseline=baseline, - ) - self.assertIn( - engine_behavioral.SKILL_DIR_KEY, task.metadata, - "the log does not say where the skill was", - ) - self.assertEqual(task.metadata[engine_behavioral.SKILL_DIR_KEY], own) - - def test_the_other_engine_records_it_too(self) -> None: - # `verify.build_task` needs `inspect_swe`, which the unit job does not - # install, so this is read rather than called -- the same compromise - # `test_the_behavioral_leg_completes_its_skill_too` makes for it. A - # log is paired without knowing which engine wrote it, so a key only - # one of them writes is a key the reader cannot rely on. - self.assertIn( - "SKILL_DIR_KEY", inspect.getsource(engine_verify.build_task) - ) - - - def test_the_verdict_is_the_same_from_any_directory(self) -> None: - # The property, asserted as a property. A relative path resolves - # against whoever is running, so resolving one here made the answer a - # fact about the reader's cwd. Standing in the repo root used to say - # contaminated and standing anywhere else said clean, for the same - # recorded call. - own = str(config.active().skill_path("run").resolve()) - argument = {"file_path": "run/SKILL.md"} - - here = os.getcwd() - self.addCleanup(os.chdir, here) - - os.chdir(self.repo.root) - from_the_repo_root = self._why(argument, skill_dir=own) - os.chdir(here) - from_anywhere_else = self._why(argument, skill_dir=own) - - self.assertEqual( - from_the_repo_root, from_anywhere_else, - "the contamination verdict moved with the reader's cwd", - ) + self.assertIn("graded", graded.tags) + self.assertIn("baseline", baseline.tags) class TestTheDetectorKeepsItsTwoAnswersApart(unittest.TestCase): @@ -8227,12 +8353,14 @@ def _call(self, name, arguments=None): def _inspect(self, sample, skill="alpha"): from skillscope.engine import routing as routing_engine - # A directory for the skill, so the room is knowable and `why` reports + # Marks for the skill, so the room is knowable and `why` reports # what this class is about rather than `UNKNOWN_ROOM`. No case here - # names a path, so nothing can match it. - return engine_behavioral._inspect_room( - routing_engine, sample, skill, f"/nonexistent/skills/{skill}" - ) + # names a path, so nothing can match them. + with mock.patch.object( + engine_behavioral, "_skill_marks", + lambda name: (f"/nonexistent/skills/{name}", f".claude/skills/{name}"), + ): + return engine_behavioral._inspect_room(routing_engine, sample, skill) def test_a_contamination_marker_is_never_reported_as_the_skill(self) -> None: # Two `other:` hits. The first sets `why`; the second used to find it @@ -8314,7 +8442,6 @@ def test_the_baseline_stages_nothing_and_is_told_nothing(self) -> None: task, seen = self._task(install=False) self.assertEqual(seen["staged"], {}, "the baseline staged the skill") self.assertEqual(seen["agent"]["skills"], [], "the agent was given the skill") - self.assertTrue(task.metadata[engine_behavioral.BASELINE_KEY]) self.assertIn("baseline", task.tags) self.assertTrue(task.name.endswith("-baseline")) @@ -8324,7 +8451,6 @@ def test_the_graded_half_gets_the_skill_both_ways(self) -> None: task, seen = self._task(install=True) self.assertEqual(list(seen["staged"]), ["run"]) self.assertEqual(len(seen["agent"]["skills"]), 1) - self.assertFalse(task.metadata[engine_behavioral.BASELINE_KEY]) self.assertIn("graded", task.tags) self.assertFalse(task.name.endswith("-baseline")) @@ -8369,15 +8495,13 @@ def setUp(self) -> None: self.repo.skill("run", dataset=tier0_dataset("run")) self.repo.activate() - def _why(self, arguments, skill="run", skill_dir=None): + def _why(self, arguments, skill="run"): from skillscope.engine import routing as routing_engine call = _FakeCall("toolu_1", function="Read") call.arguments = arguments sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) - return engine_behavioral._inspect_room( - routing_engine, sample, skill, skill_dir - )[1] + return engine_behavioral._inspect_room(routing_engine, sample, skill)[1] def test_an_unrelated_path_sharing_the_skills_name_is_not_contamination(self) -> None: self.assertIsNone(self._why({"file_path": "/run/user/1000/bus"})) @@ -8471,18 +8595,6 @@ def test_a_skill_with_no_directory_is_an_unknown_room_not_a_clean_one(self) -> N why = self._why({"file_path": "/tmp/x"}, skill="not-a-skill") self.assertEqual(why, engine_behavioral.UNKNOWN_ROOM) - def test_a_recorded_directory_answers_it_without_the_skill_on_this_machine( - self, - ) -> None: - # The point of recording it: the same log, read where the skill is not. - own = str(config.active().skill_path("run")) - self.assertIn( - "read the skill's own files", - self._why( - {"file_path": f"{own}/SKILL.md"}, skill="not-a-skill", skill_dir=own - ), - ) - class TestAWalkThatDidNotFinishCannotVouchForTheRoom(unittest.TestCase): """An interrupted walk inspected only the calls it got to. @@ -8529,9 +8641,7 @@ def function(self): messages=[_FakeMessage("m", [innocuous, _Unwalkable(), unread])], events=[], ) - _fired, why = engine_behavioral._inspect_room( - routing_engine, sample, "run", None - ) + _fired, why = engine_behavioral._inspect_room(routing_engine, sample, "run") self.assertEqual(why, engine_behavioral.UNKNOWN_ROOM) @@ -8648,6 +8758,38 @@ def test_an_unknown_cost_does_not_fold_back_as_zero(self) -> None: usage.record(input_tokens=10, calls=1) self.assertIsNone(usage.snapshot().cost_usd) + def test_a_half_that_raised_still_hands_the_global_back(self) -> None: + # The `finally`, which nothing pinned: every other test here reads + # the leg's own accumulator, which is correct whether or not the + # global was restored. Without it a raising half leaves `_current` + # pointing at that half for the rest of the process, so the second + # half's spend lands in the first half's figures and the command + # total silently stops being the sum. + # Asserted by identity, not by a total. Both ways the number comes + # out the same: the fold-back adds exactly what the leaked + # accumulator already holds, so `snapshot().input_tokens` is 15 + # whether the global was restored or left pointing at the half. What + # differs is *which object* it is. + held = [] + with self.assertRaises(RuntimeError): + with usage.leg("graded") as graded: + held.append(graded) + usage.record(input_tokens=10, calls=1) + raise RuntimeError("the engine fell over") + self.assertIsNot(usage.snapshot(), held[0].usage, + "the global was left pointing at the half that raised") + usage.record(input_tokens=5, calls=1) + self.assertEqual(usage.snapshot().input_tokens, 15) + + def test_the_wall_clock_and_the_rest_fold_back_too(self) -> None: + # `wall_time_s` is set in the same `finally`, and the report's floor + # marker and spend table both read it. + with usage.leg("graded") as graded: + usage.record(input_tokens=10, output_tokens=3, calls=1) + self.assertGreaterEqual(graded.wall_time_s, 0.0) + self.assertEqual(graded.usage.output_tokens, 3) + self.assertEqual(usage.snapshot().output_tokens, 3) + def test_an_engine_recording_from_a_log_lands_in_the_enclosing_half(self) -> None: # How the real thing records: `stats.record_log` is called deep inside # the engine, never at the leg boundary, and reaches the global by @@ -8777,6 +8919,29 @@ def test_both_halves_grade_the_same_cases(self) -> None: self.assertEqual(harness.calls[0]["cases"], harness.calls[1]["cases"]) self.assertTrue(harness.calls[0]["cases"]) + def test_the_half_with_the_skill_is_the_one_credited(self) -> None: + # The orientation of the whole command. The harness's graded half + # passes and its baseline fails, so every case must be `earns_it`; + # swapping `pair`'s arguments turns all of them into `harmful`, and + # nothing caught that because both halves used to return no outcomes + # at all. + with _staleness_harness(self) as harness: + harness.run() + totals = harness.reports[0]["totals"] + self.assertTrue(totals["earns_it"], "the graded half passed and was not credited") + self.assertFalse(totals["harmful"]) + for case in harness.reports[0]["cases"]: + self.assertEqual(case["verdict"], "earns_it") + + def test_the_time_each_half_spent_is_not_swapped_either(self) -> None: + # The same orientation, on the column where a swap reads as the + # finding this command exists to report: the baseline is the slow + # half, so `seconds_without` must be the larger number. + with _staleness_harness(self) as harness: + harness.run() + case = harness.reports[0]["cases"][0] + self.assertGreater(case["seconds_without"], case["seconds_with"]) + def test_the_baseline_half_is_given_a_config_dir_of_its_own(self) -> None: # The dir the comment calls what the whole command rests on: without # it the runner's own `~/.claude/skills` are registered beside the