diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index d87667b..f61811a 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -24,22 +24,27 @@ permissions: jobs: imports: - # One rule, not a linter. `F401` finds imports nothing uses, which is how + # Two rules, not a linter. `F401` finds imports nothing uses, which is how # dead code announces itself: two rounds of review found functions whose # last caller had been deleted, and in both cases the orphaned imports were # sitting there saying so. Twelve of them had accumulated by the time # anybody looked. # + # `F841` is the same signal one scope down, and was added after a review + # found a local left behind by a deleted report section: the list was + # still being built on every run and nothing ever read it. `F401` cannot + # see that, and the whole suite passing cannot either. + # # Deliberately narrow. Adopting a full style configuration is a change to # every file and a decision about house style; this is the one check that # pays for itself immediately and reformats nothing. - name: Unused imports + name: Unused imports and locals runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: astral-sh/ruff-action@v3 with: - args: check --select F401 skillscope tests tools + args: check --select F401,F841 skillscope tests tools unit: name: Unit tests (${{ matrix.os }}, Python ${{ matrix.python }}) diff --git a/README.md b/README.md index 5db84d9..10f4a8b 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,14 @@ SPDX-License-Identifier: MIT ![Skillscope](assets/banner.png) -A test harness for AI agent skills. Install the harness, or point a workflow at your repo. Skillscope enables three main types of tests: +A test harness for AI agent skills. Install the harness, or point a workflow at your repo. Skillscope enables four main types of tests: * **Structural**: Checks whether the skill is well structured (contains the expected files, a proper hierarchy, etc.). * **Routing**: Checks whether the skill triggers when it should and stays quiet when it shouldn't. You may also point to more than one skill here to explore how skills interfere with each other's routing. * **Behavioral**: Runs the agent end to end and uses an LLM judge to confirm the skill behaves correctly. +* **Staleness**: Runs the behavioral cases a second time with the skill withheld, and compares. Tells you whether the cases still need the skill, or whether the model has caught up and the suite is passing on its own. -All three read one dataset per skill, `/evals/evals.json`. How to write one is in [docs/authoring-evals.md](docs/authoring-evals.md). +All four read one dataset per skill, `/evals/evals.json`. How to write one is in [docs/authoring-evals.md](docs/authoring-evals.md). > [!IMPORTANT] > **Early Days**: Flags, workflow inputs, and report formats may evolve quickly as this repo takes shape. @@ -37,6 +38,7 @@ skillscope structural # no agent, no tokens skillscope structural --external # the same, plus checking external URLs skillscope behavioral --skill my-skill # needs an authenticated `claude` CLI skillscope routing --routing-room my-skill,its-neighbour # also needs a key in the environment +skillscope staleness --skill my-skill # the same key: its baseline has to be an empty room ``` `routing` needs `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` set, and refuses @@ -47,9 +49,18 @@ describe a room nobody asked for. `behavioral` installs one skill and grades what the agent produced, so a stray skill is at worst noise there, and a plain `claude login` is fine. +`staleness` is in `routing`'s camp, not `behavioral`'s, and asks the same +credential question on Claude Code. Its baseline half has to be a room with +**no** skills in it, so the runner's own are not noise there — they are the +measurement. Either set one of those variables, or use `--engine +claude-code`, whose guest has no user-level skills at all. It is Claude Code +only: unlike `routing`, which scales its refusal per harness, it refuses +any `--harness` other than `claude-code`, because there is no staleness leg +for another one yet. + ## Run it from your CI -Simply add an `evals.json` file to your skill and a workflow that points to our reusable workflow. This will triger all graders to run directly from your repo. +Simply add an `evals.json` file to your skill and a workflow that points to our reusable workflow. This will trigger the three graders the workflow runs -- `structural`, `routing` and `behavioral` -- directly from your repo. `staleness` is not one of them: it runs both halves of every case and is meant to be asked for deliberately rather than on every push. Skillscope @@ -72,6 +83,7 @@ You may also choose to customize or even disable some tests if you prefer it. Se | `skillscope structural` | Skill folders, datasets, and markdown references. | | `skillscope routing` | Which skill fires, with several installed together. | | `skillscope behavioral` | What a skill does once it has fired. | +| `skillscope staleness` | Whether those cases still need the skill, and what the agent spends without it. | | `skillscope select` | The CI plan for a change, as JSON. | | `skillscope list-skills` | The skills that have a dataset, as JSON. | | `skillscope template` | The dataset template a new skill starts from. | diff --git a/action.yml b/action.yml index 2a73c12..721ef6c 100644 --- a/action.yml +++ b/action.yml @@ -16,7 +16,7 @@ description: >- inputs: command: - description: 'The skillscope subcommand: structural, routing, behavioral, select, list-skills, template.' + description: 'The skillscope subcommand: structural, routing, behavioral, staleness, select, list-skills, template.' required: true args: description: "Further arguments for the subcommand, shell-quoted." @@ -47,8 +47,10 @@ inputs: default: "true" install-claude: description: >- - Install Node and the claude CLI. Needed by `routing` and `behavioral`, - pointless for `structural` and `select`. + Install Node and the claude CLI. Needed by `routing`, `behavioral` and + `staleness` -- staleness drives the behavioral leg for both of its + halves, so it needs the CLI exactly as much. Pointless for `structural` + and `select`. required: false default: "false" node-version: diff --git a/docs/authoring-evals.md b/docs/authoring-evals.md index 25a6b19..ee786bb 100644 --- a/docs/authoring-evals.md +++ b/docs/authoring-evals.md @@ -224,8 +224,15 @@ skillscope structural # your folder, prose, and da skillscope behavioral --skill # your skill, end to end skillscope routing --routing-room , skillscope routing --only --keep-logs logs # one case, keeping the transcript +skillscope staleness --skill # do your cases still need the skill? ``` +`staleness` is the one aimed at this file's subject. It runs your behavioral +cases twice, once with the skill installed and once without, and reports which +of them passed either way. Those are cases that are not evidence about the +skill — either it has stopped mattering, or the case never tested it. It costs +two runs, so it belongs on a schedule rather than in the loop. + `structural` is instant and needs no agent. Everything else needs the `claude` CLI authenticated, plus whatever your own cases need. Flags, what each check asserts, the routing bar, and how to read a report are in diff --git a/docs/harnesses.md b/docs/harnesses.md index 4c3be4b..3d70e0c 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -145,4 +145,8 @@ Claude Code, and a harness other than it cannot yet use them: each agent is defined once. Until it does, a harness other than Claude Code has no sandboxed leg; * **`.claude/skills`** as the path routing treats as a skills inspection, rather - than a harness's own `skills_dir`. + than a harness's own `skills_dir`; +* **`staleness`**: its baseline is only a measurement if the room is empty, and + the check that establishes that reads Claude Code's credentials and no other + harness's, so it refuses any `--harness` other than `claude-code` rather + than scaling the refusal per harness. diff --git a/docs/usage.md b/docs/usage.md index ee4c527..7b08688 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -18,6 +18,7 @@ CLI flag, and every workflow input. For writing a skill's dataset, see * [References](#references) * [Routing, and the bar it is held to](#routing-and-the-bar-it-is-held-to) * [Hardware a skill needs](#hardware-a-skill-needs) +* [Is the skill still earning its place?](#is-the-skill-still-earning-its-place) * [In CI: one job](#in-ci-one-job) * [In CI: the full pipeline](#in-ci-the-full-pipeline) * [Versions](#versions) @@ -249,6 +250,135 @@ Legs with a scoped environment run as a separate job, because a job's credentials are fixed before its matrix expands. A repo that declares no scoped environment gets one matrix, labels and all. +## Is the skill still earning its place? + +A behavioral run says a skill's cases pass. It cannot say they pass *because +of* the skill, and those are different claims: an expectation any competent +agent meets is a measurement of the model with the skill's name on it. + +`skillscope staleness` runs the same cases twice, once with the skill +installed and once with nothing installed, and reports which ones moved. + +```bash +skillscope staleness --skill my-skill +``` + +On the default engine this needs `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` +in the environment and refuses to start without one, the same way `routing` +does and for a stronger version of the same reason: the key is what lets the +run redirect the CLI away from the runner's own config dir, and the baseline +half has to be a room with no skills in it. Without it, every skill installed +on the machine joins the baseline, and a case one of them solves there is +reported as a case the skill under test was not needed for. A plain `claude +login` is enough for `behavioral` and is not enough here. `--engine +claude-code` needs no key for this, since its guest has no user-level skills +at all. + +The report leads with what each half cost, because that is the cheaper +question and often the more useful answer. An agent holding the skill goes +straight to the work; an agent without it explores. On the first real run of +this command the graded half finished in 2m23s and the baseline was still +going half an hour later, with every case passing on both sides -- a skill +plainly earning its keep, which a verdict table alone reports as `no_signal` +throughout. + +| Half | Wall time | Model calls | Tokens | Cost | +| --- | --- | --- | --- | --- | +| with the skill | 2m23s | 37 | 184220 | n/a | +| nothing installed | 28m00s+ | 214 | 1044901 | n/a | + +A `+` marks a half the `--timeout` stopped: that row is a floor, not a +measurement. Cost reads `n/a` unless the provider supplies pricing, which a +gateway generally does not and which inspect's own registry supplies for no +model at all; `SKILLSCOPE_MODEL_PRICING` is the way in. **A missing cost is a +missing number and not a zero** -- dollars arrive with a run's final event, +which a half the bound stopped never reaches, so the half that spent *more* +is the likelier one to show nothing. Tokens and wall clock are the reliable +columns. + +Then the verdicts. Three, and an error state: + +| Verdict | Means | +| --- | --- | +| `earns_it` | passed with the skill, failed without it -- the case discriminates | +| `harmful` | failed with the skill, passed without it -- the skill steered the agent wrong | +| `no_signal` | came out the same both ways | +| `ungraded` | nothing could be concluded: every attempt errored on one side or both, a grader never answered, a case was missing from a side, or the baseline was not the empty room it claimed to be | + +Three rather than more. Earlier versions split out whether the skill fired, +whether a case claimed anything positive at all, and whether a shared failure +was a broken case or a hard one. All three are still reported, as counts and +notes beside the verdict rather than as verdicts of their own. None of them +was a different answer to the question this command asks; each was one of +these answers with a caveat attached. + +A verdict says what changed, not why. So the graded side also records whether +the agent *reached for* the skill, using the same detector the routing leg +uses: installing a skill does not make an agent use it, and "installed and the +outcome differed" is a weaker claim than "the skill fired and the outcome +differed". A case that moved without the skill firing is still `earns_it` -- +the outcome did differ -- but it is counted and flagged above the case table, +so the headline share can be discounted by a reader who cares. Guard rails, +`unexpected_behavior`, are kept out of the verdict entirely, since a good one +holds whether or not the skill is installed; they are counted on their own, +and in both directions. A guard broken *with* the skill installed is the +serious finding. A guard the **baseline** broke and the installed side held +is the opposite one, and the report says so: the skill stopped the agent +doing the forbidden thing. Both sides can reach the same answer with only one +of them wrecking something on the way, and no verdict in the table can +express that. The first real run of this command found one unprompted, an +unaided agent attempting a `sudo` write on a case the skill kept read-only. + +`no_signal` is the verdict worth acting on, and it is three findings wearing +one name. The pass counts and the advice column say which you are holding: + +* **both sides passed** -- the agent already does this unaided, so the case is + not evidence about the skill. Either the skill has stopped mattering or the + case never tested it, and the second is more common, which makes this as + much a check on the dataset as on the skill. +* **both sides failed** -- a broken case or a hard one. Nothing here has been + shown to work well enough to then ask whether the skill is why, so this + points at the case rather than the skill. +* **a side disagreed with itself** -- only possible above one attempt. The + baseline managed it at least once, which is not the same finding as a case + both sides managed every time. Re-run before reading anything into it. + +A case that claims nothing positive at all is `no_signal` too, since neither +side can meet a claim that was never made. The advice column says so rather +than calling it broken: its guard rails were still checked, and the case is +working exactly as written. + +### Spending less on a run + +Both halves of a comparison are agent passes, and the baseline is the slower +one: an agent holding the skill goes straight to the work, and an agent +without it explores. Three flags decide what a run costs. + +| Flag | Default | Effect | +| --- | --- | --- | +| `--attempts N` | 1 | Run every case N times on each side. Costs N times as much. A side counts as having passed when **any** of its attempts did, so a higher N is a harder question rather than a steadier reading of the same one: `earns_it` at three attempts needs the baseline to have failed all three times, and a baseline that passes once in three takes the case to `no_signal`. Verdicts compare with verdicts from runs at the same N, not across different ones. What you gain is the counts printed beside each verdict: `3/3` against `0/3` is a result, `3/3` against `1/3` is a baseline that managed it once, and `2/3` against `1/3` is two sides that both disagreed with themselves. The advice column calls that last one out rather than letting it read as a dead case. | +| `--max-cases N` | every case | Grade only the first N cases of each selected skill, after `--only` and `--extended` have chosen them. Both sides run the same N, and the report says the run did not cover the whole suite. | +| `--concurrency N` | inspect's default | How many cases may be in flight at once. Where the wall clock goes on the baseline half. Raise it as far as the engine's sandboxes and the model's rate limits allow. | + +Budget more wall clock than a `behavioral` run of the same cases, and more +than double it. `--timeout` defaults to twice what the single-pass commands +use for that reason. Both passes share one `--timeout`, and the baseline is +reliably the slower half: the agent holding the skill goes straight to the +work, while the one without it explores. On a two-case skill the graded pass +took 2m23s and the baseline had not finished its second case half an hour +later. A bound that expires partway reports the unreached cases as `ungraded`, +after the graded half has already been paid for. + +No verdict it reaches fails a run, for the reason the `claude-code` leg does +not gate either: the agent is nondeterministic on both sides, so a single pair +is a place to look rather than a verdict. An expired `--timeout` or a refused +run still exits non-zero, because those are statements about the run rather +than about a skill. [`tools/benchmark_engines.py`](../tools/benchmark_engines.py) +is the shape of the honest version of this comparison — it measures agreement +against how much one engine already disagrees with itself — and a staleness +number deserves the same scepticism. It also costs twice what `behavioral` +costs, which is why no workflow runs it for you. + ## Which engine grades a run Two choices decide what runs the cases, and they are separate: `--engine` is @@ -304,8 +434,14 @@ routing leg at all unless one of the harness's credential variables is set (see [harnesses.md](harnesses.md): `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` for Claude Code, `COPILOT_GITHUB_TOKEN`, `GH_TOKEN` or `GITHUB_TOKEN` for Copilot CLI), which is what lets it redirect the CLI's -config dir away from the runner's own. Refusing beats being -quietly wrong about every case. +config dir away from the runner's own. + +`staleness` is stricter, and not on the same terms. It has no leg for any +harness but Claude Code yet, so it refuses any `--harness` other than +`claude-code` whatever credentials are present, rather than scaling the +refusal per harness the way routing does. On Claude Code it then asks routing's question as well, because +there the empty room is not one input to the measurement, it is the +measurement. Refusing beats being quietly wrong about every case. Both legs stop at the decision, by different routes. `claude-code`'s calls cross inspect's bridge, so the one that reveals the decision is declined diff --git a/pyproject.toml b/pyproject.toml index 70a26a8..0544635 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ authors = [{ name = "Advanced Micro Devices, Inc." }] # # It was an extra while `legacy` existed: that engine was standard library only, # so a repo that had not migrated installed nothing new. `legacy` is gone, and -# keeping the extra would ship a tool whose two graded commands both fail on a +# keeping the extra would ship a tool whose graded commands all fail on a # plain `pip install` -- which is what CI found, by running the suite without # extras and getting `ModuleNotFoundError: inspect_ai` from the default engine. # diff --git a/skillscope/behavior.py b/skillscope/behavior.py index b382b1d..1b0caf3 100644 --- a/skillscope/behavior.py +++ b/skillscope/behavior.py @@ -29,11 +29,29 @@ class BehaviorOutcome: elapsed_s: float checks: list[dict] = field(default_factory=list) error: str | None = None + # The skill the agent actually reached for, if any. A behavioral run + # installs a skill but cannot make the agent use it, and until this was + # recorded a case that passed was credited to a skill that may never have + # fired. `skillscope staleness` is what the distinction is for: "installed + # and the outcome differed" is a weaker claim than "the skill fired and the + # outcome differed", and only the second is evidence about the skill. + activated: str | None = None + # Why this run was not the room it claimed to be: another skill fired, or + # the agent read the skill's own source off disk. Harmless noise in a + # behavioral run, which grades what the agent produced. Fatal to a + # staleness baseline, whose whole claim is that nothing was available. + contaminated: str | None = None # Set when the failure was the model provider's rather than the skill's. # A behavioral case that failed on a gateway 504 is a skill nobody # measured, and reporting it beside one that genuinely failed its # expectations invites exactly the wrong conclusion. degraded: bool = False + # Which run of this case this is, where `skillscope staleness --attempts` + # asked for more than one. Always 1 elsewhere. It exists so two answers to + # the same case can be told apart: `staleness` groups a side's attempts by + # case id, and the guard that refuses a case two logs both claim to answer + # has to know a repeat from a collision. + attempt: int = 1 # -------------------------------------------------------------------------- diff --git a/skillscope/cli.py b/skillscope/cli.py index bc9c406..61a7aa4 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -5,7 +5,7 @@ """The one entry point for skill evals. Every skill owns a single dataset at ``/evals/evals.json``. This runner -reads those datasets and grades them with three commands: +reads those datasets and grades them with four commands: * ``structural`` -- every skill folder, every dataset, and every reference the skill's markdown makes. No agent, no tokens, instant. @@ -18,13 +18,17 @@ grades ``expected_behavior`` / ``unexpected_behavior`` / ``logs_contain`` / ``files_exist``. Only runs evaluations that assert something beyond the routing decision. + * ``staleness`` -- runs ``behavioral``'s cases twice, once with the skill + installed and once without, and compares. Answers a question the other + three cannot: whether the cases still need the skill, or whether the model + has caught up and the suite is passing on its own. A skill may also ship ``/evals/extended_evals.json``, in the same format -and under no coverage requirement of its own. Both graded commands include it +and under no coverage requirement of its own. The graded commands include it by default; ``--no-extended`` grades the required dataset alone. -The prompt is written once and both graded commands read it, which is the whole -point: the alternative is a central routing prompt set plus a separate +The prompt is written once and every graded command reads it, which is the +whole point: the alternative is a central routing prompt set plus a separate per-skill test file that re-asserts routing with a substring match on the transcript. @@ -52,6 +56,10 @@ # one case, keeping the raw transcript skillscope routing --only qwen-on-mi300x --keep-logs eval-logs + # whether the skill is earning its place: every case run with it and + # without it, twice the agent passes of a behavioral run + skillscope staleness --skill serving-llms-on-epyc + # what CI should run for a change skillscope select --since BASE HEAD @@ -68,6 +76,7 @@ import json import os import sys +import tempfile import time from pathlib import Path @@ -79,6 +88,7 @@ engine, references, routing, + staleness, structure, usage, ) @@ -424,17 +434,29 @@ def _empty_room(args: argparse.Namespace) -> None: ) -def _fail_if_expired() -> int | None: - """Non-zero when the command's ``--timeout`` has elapsed or stopped the run. +def _within_deadline() -> bool: + """Whether the command's ``--timeout`` is still unspent. - Both, because the graceful stage fires `GRACEFUL_RESERVE_S` *before* the - wall. inspect then returns from `eval()` normally, so a run it stopped - reaches the gate with the clock still short of `expired()` -- and was - reported as "every case errored" instead of the overrun it was. + Both conditions, because the graceful stage fires `GRACEFUL_RESERVE_S` + *before* the wall. inspect then returns from `eval()` normally, so a run + it stopped is back here with the clock still short of `expired()` -- + which is why `interrupted` is asked as well as the time. + + Asked between the two halves of a staleness run as well as at the end, so + a half the bound stopped can be reported as a floor rather than as a + measurement. A baseline that was still going when the clock ran out is + the finding, and a report that cannot tell it from a baseline that + finished states a weaker thing than the run actually established. """ bound = deadline.active() - if bound is None or not (bound.expired() or bound.interrupted): + return bound is None or not (bound.expired() or bound.interrupted) + + +def _fail_if_expired() -> int | None: + """Non-zero when the command's ``--timeout`` has elapsed or stopped the run.""" + if _within_deadline(): return None + bound = deadline.active() print(f"error: {bound.message()}", file=sys.stderr) return 1 @@ -442,7 +464,7 @@ def _fail_if_expired() -> int | None: def _prepare_graded_run( args: argparse.Namespace, scope: list[str] | None = None ) -> list[str]: - """Structural checks, model pin, and API reachability. Shared by both graders. + """Structural checks, model pin, and API reachability. Shared by the graders. The structural gate reads `scope`, defaulting to the skills ``--skill`` selected -- the ones about to be graded, and nobody else. A routing run @@ -484,8 +506,8 @@ def _prepare_graded_run( ) # Also before the probe, and for the same reason: a malformed price # table should fail the command, not one sample into a graded run. - # Registered here rather than in the engine because both graded - # commands come through this function and inspect's registry is + # Registered here rather than in the engine because every graded + # command comes through this function and inspect's registry is # process-global -- setting it twice would be the same work. from .engine import pricing as engine_pricing @@ -537,9 +559,11 @@ def _require_hook_support(engine: str, skills: list[str], command: str) -> None: run perfectly well. Routing is exempt on every engine: it installs the skills, asks which one - fires, and executes nothing. + fires, and executes nothing. Staleness is not: it drives the same + `build_task` on the same cases, twice, so a `check` it skipped is skipped + on both halves and the comparison is between two runs that both lied. """ - if command != "behavioral": + if command not in ("behavioral", "staleness"): return from .engine import hooks as engine_hooks @@ -784,61 +808,136 @@ def cmd_routing(args: argparse.Namespace) -> int: ) -def cmd_behavioral(args: argparse.Namespace) -> int: - skills = _prepare_graded_run(args) - subject = harnesses.get(args.harness) - started = time.time() +def _gradable_cases(args: argparse.Namespace, skills: list[str]) -> list: + """The cases a behavioral-style run grades: selected, filtered, assertive. + Shared so `behavioral` and `staleness` cannot drift apart about what + `--only` or `--extended` select. A staleness run comparing a different set + of cases than the behavioral run it is read beside would be the one bug + neither report could show you. + """ cases = [] for skill in skills: cases.extend(datasets.load_dataset(skill, extended=args.extended)) if args.only: + # Applied across every selected skill at once, because it refuses a + # selection that matched nothing. Per skill, `--only` naming a case of + # one skill would fail the run on the next skill's empty match. cases = datasets.filter_cases(cases, args.only) - gradable = [c for c in cases if c.has_behavior] - - if not gradable: - print( - "[behavioral] no evaluation in the selected skill(s) asserts " - "anything beyond routing, so there is nothing to grade. Add " - "`expected_behavior` / `unexpected_behavior` / `logs_contain` / " - "`files_exist` to a triggering evaluation." - ) - return 0 + cases = [c for c in cases if c.has_behavior] + # Capped here rather than through inspect's own `limit`, which applies per + # task, and a run builds one task per skill: the same number would mean + # "the first N cases" on a one-skill run and "N per skill" on a four-skill + # one. Capped after the filters, so `--only` and `--max-cases` compose the + # way a reader would expect. + # + # And capped once, in the shared selector, because the two halves of a + # staleness run have to grade the same cases. A cap applied anywhere + # downstream could hand the baseline a different set, and a comparison + # between two runs that answered different questions is worse than no + # comparison: nothing in the report would say so. + # + # Per skill rather than over the whole run, so a four-skill run does not + # spend its whole budget on the first skill's dataset and report the other + # three as having nothing to grade. + cap = getattr(args, "max_cases", 0) + if cap: + kept: list = [] + for skill in skills: + kept.extend([c for c in cases if c.skill == skill][:cap]) + return kept + return cases + + +def _run_behavioral( + args: argparse.Namespace, + skills: list[str], + gradable: list, + *, + install: bool = True, + config_dir: Path | None = None, +) -> list: + """Run the behavioral cases on the selected engine. + + `install` False runs them with no skill staged, which is the baseline half + of `cmd_staleness`. Both halves go through here rather than through two + similar call sites, so the only difference between the run being measured + and the run it is measured against is the flag. + + `attempts` and `concurrency` are read off `args` rather than passed, so + the two halves cannot be given different ones. A baseline attempted fewer + times than the graded half is not a baseline, and the counts the report + prints would be comparing different denominators without saying so. + """ from .engine import behavioral as inspect_behavioral from .engine import models as engine_models + attempts = getattr(args, "attempts", 1) + concurrency = getattr(args, "concurrency", 0) + if args.engine == "claude-code": from .engine import verify as runner - outcomes = runner.run( - skills, gradable, engine_models.resolve(args.model), args.effort + return runner.run( + skills, + gradable, + engine_models.resolve(args.model), + args.effort, + install=install, + attempts=attempts, + concurrency=concurrency, ) - elif args.engine == "claude-code-no-sandbox": + if args.engine == "claude-code-no-sandbox": # The real CLI, driven on the host, inside inspect's framework. # `--model` stays the CLI's own alias here: this one does not go # through an inspect model provider. from .engine import no_sandbox + # The harness is read here rather than passed in, for the reason + # `attempts` and `concurrency` are: both halves of a staleness run go + # through this function, so reading it once is what stops the two + # halves being given different agents. + subject = harnesses.get(getattr(args, "harness", None)) no_sandbox.require_local(subject) - outcomes = inspect_behavioral.run( + return inspect_behavioral.run( skills, gradable, engine_models.resolve(args.model), args.effort, solver_factory=lambda d: no_sandbox.harness_solver( - subject, args.model, args.effort, d + subject, args.model, args.effort, d, config_dir, install=install ), + baseline=not install, + attempts=attempts, + concurrency=concurrency, ) - else: - # Not a fallthrough. An engine in ENGINES with no branch here is a - # bug this dispatch has had once already, and a silent default is - # what let it survive: runs asked for one agent, got another, and - # the report named the one they had asked for. - raise SystemExit( - f"error: --engine {args.engine} has no behavioral leg. " - f"This is a bug in skillscope, not in how it was called." + # Not a fallthrough. An engine in ENGINES with no branch here is a + # bug this dispatch has had once already, and a silent default is + # what let it survive: runs asked for one agent, got another, and + # the report named the one they had asked for. + raise SystemExit( + f"error: --engine {args.engine} has no behavioral leg. " + f"This is a bug in skillscope, not in how it was called." + ) + + +def cmd_behavioral(args: argparse.Namespace) -> int: + skills = _prepare_graded_run(args) + started = time.time() + + gradable = _gradable_cases(args, skills) + + if not gradable: + print( + "[behavioral] no evaluation in the selected skill(s) asserts " + "anything beyond routing, so there is nothing to grade. Add " + "`expected_behavior` / `unexpected_behavior` / `logs_contain` / " + "`files_exist` to a triggering evaluation." ) + return 0 + + outcomes = _run_behavioral(args, skills, gradable) summary = behavior.summarize( outcomes, @@ -863,6 +962,167 @@ def cmd_behavioral(args: argparse.Namespace) -> int: return 0 +def cmd_staleness(args: argparse.Namespace) -> int: + """Run every behavioral case twice, with the skill and without it. + + Both passes run under the one `--timeout`, so a deadline that expires + partway leaves the rest of the second pass unmeasured -- those cases are + reported `ungraded` rather than as a skill that made no difference. + + Budget for a baseline that runs *longer than the graded pass*, not the + same. An agent holding the skill goes straight to the work; an agent + without it explores -- building throwaway virtualenvs, dry-running + installs, reading around the problem. Measured on a two-case skill: the + graded pass finished both cases in 2m23s and the baseline was still going + on the second after 30 minutes. Twice a `behavioral` bound is the floor, + not the figure -- which is why this parser defaults `--timeout` to twice + what the single-pass commands use. On the shared 900s default those + numbers left the slower half ~637s of a measured 1800s+ need, so + truncation was the expected outcome on this command's own example rather + than a risk it ran. + + Always exits 0 on a completed run. The comparison is nondeterministic on + both sides, so a verdict here is somewhere to look rather than a build + failure -- see `skillscope/staleness.py`. A deadline that expired still + fails, because that is a statement about the run rather than about a skill. + """ + started = time.time() + + asked = getattr(args, "harness", None) or os.environ.get("SKILLSCOPE_HARNESS") + if asked and asked != harnesses.DEFAULT: + # The one place this command cannot be harness-neutral yet. Its + # baseline half is only a measurement if the room is empty, and the + # check that establishes that -- `routing.can_isolate_config` below -- + # reads Claude Code's credential variables and no other harness's. Run + # against a different agent with an Anthropic key happening to sit in + # the environment, it would report an isolated room it had not + # isolated, which is the one error this whole command exists to avoid + # making. Refused rather than run on a baseline nobody can vouch for. + raise SystemExit( + f"error: --harness {asked} has no staleness leg yet. The baseline " + "half has to be a room with no skills in it, and the check that " + f"establishes that reads {harnesses.DEFAULT}'s credentials rather " + "than this harness's -- so a run here would claim an empty room " + f"it never checked. Use --harness {harnesses.DEFAULT}, or " + "`routing` and `behavioral`, which are harness-neutral." + ) + + if args.attempts < 1: + raise SystemExit( + f"error: --attempts {args.attempts} would run nothing. The " + "command compares a side that ran against a side that ran; one " + "attempt each is the minimum that produces a comparison." + ) + if args.max_cases < 0 or args.concurrency < 0: + raise SystemExit( + "error: --max-cases and --concurrency count things, so they " + "cannot be negative. 0 means no cap on either." + ) + + skills = _prepare_graded_run(args) + gradable = _gradable_cases(args, skills) + + if not gradable: + print( + "[staleness] no evaluation in the selected skill(s) asserts " + "anything beyond routing, so there is nothing to compare. A " + "staleness run needs cases that say what the skill should do." + ) + return 0 + + # The runner's own `~/.claude/skills` are registered beside the staged + # ones, so without this the baseline is not an empty room -- it is this + # machine's skills minus the one under test. `routing` has refused to run + # without isolation for the same reason; staleness needs it more, because + # the baseline being empty is the entire measurement rather than one + # input to it. + if args.engine == "claude-code-no-sandbox" and not routing.can_isolate_config(): + raise SystemExit( + "error: staleness cannot run on --engine claude-code-no-sandbox " + f"without {' or '.join(routing.ENV_CREDENTIALS)}. The baseline " + "half has to be a room with no skills in it, and redirecting the " + "CLI away from the runner's own config dir only works when auth " + "comes from the environment. Without it every skill installed on " + "this machine joins the baseline, and a case it solves there " + "would be reported as one the skill under test was not needed " + "for.\n Set one of those, or use --engine claude-code, whose " + "guest has no user-level skills at all." + ) + + + passes = ( + "run twice" if args.attempts <= 1 + else f"run {args.attempts} time(s) on each side" + ) + print(f"[staleness] {len(gradable)} case(s), {passes}", flush=True) + # A directory per half, not one shared between them. The contamination + # check reads what the agent did, so it cannot see state the CLI itself + # writes into its config dir; were the graded half to leave anything + # there that the baseline half could pick up, the baseline would no + # longer be the empty room the whole command rests on. Nothing is known + # to do this, but a second temporary directory costs nothing and the + # question stops needing an answer. + with ( + tempfile.TemporaryDirectory(prefix="skillscope-staleness-with-") as graded_dir, + tempfile.TemporaryDirectory(prefix="skillscope-staleness-without-") as bare_dir, + ): + host = args.engine == "claude-code-no-sandbox" + # Billed per half. The graded half runs first and the two are + # sequential, so each scope covers exactly one half's spend -- and + # `finished` is asked straight after each, because the bound they + # share is what stops the second one. + with usage.leg("graded") as graded_leg: + with_skill = _run_behavioral( + args, skills, gradable, + config_dir=Path(graded_dir) if host else None, + ) + graded_leg.finished = _within_deadline() + with usage.leg("baseline") as baseline_leg: + without = _run_behavioral( + args, skills, gradable, install=False, + config_dir=Path(bare_dir) if host else None, + ) + baseline_leg.finished = _within_deadline() + + comparisons = staleness.pair(with_skill, without) + summary = staleness.summarize( + comparisons, + { + # Which agent graded, where it ran, and which build of it, the + # way every other graded report says it. Hand-written before, + # which spelled the engine without `_engine_label`'s + # normalisation and left out `harness` and `harness_version` + # altogether -- a reader holding a routing or behavioral report + # beside a staleness one found two keys missing and a third in + # a different alphabet. + **_leg_meta(args), + "model": args.model, + "effort": args.effort, + "skills": skills, + "extended": args.extended, + "attempts": args.attempts, + # Recorded only when it bound. A `0` in the report would read as a + # cap of nothing rather than as no cap, and `render_markdown` + # warns about a partial run on the presence of this key. + **({"max_cases": args.max_cases} if args.max_cases else {}), + **({"concurrency": args.concurrency} if args.concurrency else {}), + "wall_time_s": round(time.time() - started, 1), + "timeout": args.timeout, + "github_run_id": os.environ.get("GITHUB_RUN_ID"), + # In run order, which is also the order the report prints them. + # The totals below stay the sum of the two, so every other + # report's meta keeps the shape it had. + "legs": [graded_leg.as_meta(), baseline_leg.as_meta()], + **usage.snapshot().as_meta(), + **_sandbox_meta(args), + }, + ) + _write_report(summary, staleness.render_markdown(summary), args, "staleness") + if (code := _fail_if_expired()) is not None: + return code + return 0 + + # -------------------------------------------------------------------------- # Argument parsing # -------------------------------------------------------------------------- @@ -921,8 +1181,14 @@ def _add_routing_room_argument( ) -def _add_graded_arguments(parser: argparse.ArgumentParser) -> None: - """Flags shared by the routing and behavioral commands.""" +def _add_graded_arguments( + parser: argparse.ArgumentParser, *, timeout: float = deadline.DEFAULT_TIMEOUT_S +) -> None: + """Flags shared by the graded commands: routing, behavioral and staleness. + + `timeout` is threaded through rather than overridden afterwards so the + `--help` text matches the default in force. See `_add_timeout_argument`. + """ _add_skills_argument(parser) parser.add_argument( "--skill", @@ -997,18 +1263,28 @@ def _add_graded_arguments(parser: argparse.ArgumentParser) -> None: "comparable. Default: claude-code, or $SKILLSCOPE_HARNESS." ), ) - _add_timeout_argument(parser) + _add_timeout_argument(parser, timeout) -def _add_timeout_argument(parser: argparse.ArgumentParser) -> None: +def _add_timeout_argument( + parser: argparse.ArgumentParser, default: float = deadline.DEFAULT_TIMEOUT_S +) -> None: + """The whole-command deadline, with a help string that knows its default. + + `default` is a parameter rather than a `set_defaults` call afterwards + because the help text is built here and argparse does not revisit it. + `staleness` overrode the value and left the help saying 900, three lines + under a description announcing that it defaults to twice the usual -- a + screen of `--help` contradicting itself. + """ parser.add_argument( "--timeout", type=float, - default=deadline.DEFAULT_TIMEOUT_S, + default=default, help=( "Seconds before the whole command is abandoned. Shared by " - "structural, routing, and behavioral. Default: 900 (15 minutes). " - "0 disables." + "structural, routing, behavioral and staleness. " + f"Default: {default:g} ({default / 60:g} minutes). 0 disables." ), ) @@ -1193,6 +1469,80 @@ def build_parser() -> argparse.ArgumentParser: _add_graded_arguments(behavioral_parser) behavioral_parser.set_defaults(handler=cmd_behavioral) + staleness_parser = commands.add_parser( + "staleness", + help="Whether a skill's cases still need the skill.", + description=( + "Run every behavioral case twice -- once with the skill installed, " + "once with nothing installed -- and report which cases moved. A " + "case that passes either way is not evidence about the skill. " + "Reports only: a verdict here never fails a run, though an " + "expired deadline or a refused run still exits non-zero. It costs " + "twice what `behavioral` costs and wants more than twice the " + "--timeout, which is why --timeout defaults to twice the usual " + "here: both passes share one bound, and the baseline is the " + "slower half because an agent without the skill explores instead " + "of going straight to the work." + ), + ) + # Twice the shared bound, because the command is two passes under one. + # Passed into `_add_graded_arguments` rather than set afterwards, so the + # `--help` text is built from the number actually in force: a + # `set_defaults` override changes the value and leaves the help saying + # whatever it was built with. The measurement behind the number is in + # `cmd_staleness`: a two-case skill finished the graded pass in 2m23s and + # had not finished the baseline after 30 minutes, so the shared 900s gave + # the slower half ~637s of a measured 1800s+ need. + _add_graded_arguments(staleness_parser, timeout=2 * deadline.DEFAULT_TIMEOUT_S) + staleness_parser.add_argument( + "--attempts", + type=int, + default=1, + metavar="N", + help=( + "Run every case N times on each side instead of once. One pair of " + "runs is a lead rather than a result -- an agent disagrees with " + "itself between runs, so a case can pass with the skill and fail " + "without it for no reason the skill is responsible for. Repeats " + "are what let a reader weigh a verdict: the report prints how " + "many attempts passed on each side, and 3/3 against 0/3 is not " + "3/3 against 1/3. A side counts as having passed when any of its " + "attempts did, which makes a higher N a higher bar rather than a " + "steadier reading of the same one: earns_it at three attempts " + "needs the baseline to have failed all three times, and a " + "baseline that passes once takes the case to no_signal. So " + "verdicts compare across runs at equal N and not otherwise. " + "Costs N times what it costs now, on both sides. Default: 1." + ), + ) + staleness_parser.add_argument( + "--max-cases", + type=int, + default=0, + metavar="N", + help=( + "Grade only the first N cases of each selected skill, after " + "--only and --extended have chosen them. For trying the command " + "on a large suite without paying for all of it. Both sides run " + "the same N, and the report says the run did not cover the whole " + "suite. Default: 0, every case." + ), + ) + staleness_parser.add_argument( + "--concurrency", + type=int, + default=0, + metavar="N", + help=( + "How many cases may be in flight at once. This command's baseline " + "half is the slow one -- an agent with no skill explores -- and " + "running its cases one after another is where the wall clock " + "goes. Raise it as far as the engine's sandboxes and the model's " + "rate limits will take. Default: 0, inspect's own default." + ), + ) + staleness_parser.set_defaults(handler=cmd_staleness) + select_parser = commands.add_parser( "select", help="Emit the CI plan for a change, as JSON.", diff --git a/skillscope/datasets.py b/skillscope/datasets.py index aea25f9..4464404 100644 --- a/skillscope/datasets.py +++ b/skillscope/datasets.py @@ -38,6 +38,12 @@ with a substring match is a worse version of a check this module already models as a field. +``staleness`` adds no third thing to write. It grades the behavioral cases +above a second time with the skill withheld, which makes it a check on this +dataset as much as on the skill: a case that passes either way is not +evidence about the skill, whether because the skill stopped mattering or +because the case was never evidence in the first place. + ``skill_should_trigger: false`` makes the evaluation routing-only. No skill loads for it, so there is no behavioral phase to hang an assertion or a staged workspace off, and those fields are rejected rather than silently ignored: diff --git a/skillscope/engine/behavioral.py b/skillscope/engine/behavioral.py index 06ca60f..3cd5e16 100644 --- a/skillscope/engine/behavioral.py +++ b/skillscope/engine/behavioral.py @@ -16,10 +16,11 @@ from __future__ import annotations +import re import sys from pathlib import Path -from .. import agent, config, deadline, routing as routing_core +from .. import agent, config, deadline, harness as harnesses, routing as routing_core from ..behavior import BehaviorOutcome from ..datasets import Case from . import convert, hooks, models, sandbox as sandbox_spec, scorers, stats @@ -85,6 +86,7 @@ def build_task( ctx: dict | None = None, *, solver_factory, + baseline: bool = False, ): """One inspect `Task` per skill: its cases, its skill installed, its scorer. @@ -96,6 +98,11 @@ def build_task( Required, and keyword-only. It was optional while a built-in react agent was the default, and a caller that forgot it silently graded a different agent than it asked for. + + `baseline` only names the task. Whether the skill is actually staged is + the solver's business, so the flag and the factory have to agree -- the + name is what makes a disagreement visible in the log directory rather + than invisible in the numbers. """ from inspect_ai import Task @@ -110,7 +117,8 @@ def build_task( bound = deadline.active() return Task( - name=f"behavioral-{skill}", + name=f"behavioral-{skill}" + ("-baseline" if baseline else ""), + tags=["skillscope", "baseline" if baseline else "graded"], dataset=samples, setup=hooks.setup_solver(hook, skill, cases), solver=solver_factory(skill_dir), @@ -138,6 +146,463 @@ def _failed(skill: str, cases: list[Case], detail: str) -> list[BehaviorOutcome] ] +def _normalise(text: str) -> str: + """Lowercased, separators flattened, no-op segments collapsed. + + The collapse is textual and needs no cwd: `a/b/../c` is `a/c` whoever is + asking, and so are `a//c` and `a/./c`. Only a `..` with a real segment in + front of it is removed, so a leading `../x` is left alone -- resolving + that *would* need the agent's cwd, which no tool-call record carries. + + Without it, `/other/../run/SKILL.md` names the skill and matched + nothing, and a baseline that read the skill by that spelling was reported + as a clean room. `//` and `/./` are the same miss in a cheaper spelling: + a shell that builds a path by joining two pieces produces them without + anyone typing them, and both name the skill exactly. + + A trailing separator goes with them, which costs nothing: `_mentions` + treats the end of the string as a boundary, so `/run/` and + `/run` match the same mark. + """ + flat = text.replace("\\", "/").lower() + # `/.` covers `/./`, `/../` and a trailing `/.` or `/..`; `//` covers a + # repeated separator anywhere. + if "//" not in flat and "/." not in flat: + return flat + out: list[str] = [] + for index, part in enumerate(flat.split("/")): + # A repeated or trailing separator. The leading empty segment of an + # absolute path is kept: it is what makes the path absolute. + if part == "" and index and out: + continue + if part == "." and out: + continue + if part == ".." and out and out[-1] not in ("", "..") and ":" not in out[-1]: + out.pop() + continue + out.append(part) + return "/".join(out) + + +# `~/x` and `$HOME/x` spell the same file as the absolute path, and a mark is +# always absolute. Anchored so it only fires where a path can start, and only +# where one actually follows. +_HOME = re.compile(r"(? str: + """`text` again with `~/` and `$HOME/` written out, where they appear. + + Separate from `_normalise` because it is the one expansion that is not + context-free. It resolves against *this* process's home, which is the + agent's own on the live path -- `no_sandbox` runs it on this host -- and + is a guess on a rebuild of logs written somewhere else. Worth making + anyway: the alternative is reading `cat ~/dev/x/skills/run/SKILL.md` as + a clean room, and the answer it buys is voided-or-unknown rather than a + claim about the skill. + + Applied to whole command strings, not just to arguments that are paths. + The expansion before this one fired only on a value that started with + `~` and had no whitespace in it, which is a bare path -- so every `~` + inside a Bash command line, which is where an agent actually types one, + went unexpanded. + """ + if "~" not in text and "$" not in text: + return text + try: + home = _normalise(str(Path.home())) + except (Exception, SystemExit): # noqa: BLE001 -- no home, nothing to do + return text + return _HOME.sub(home, text) if home else text + + +# Shapes that read a whole directory rather than one named file. A call +# carrying one of these *and* naming a directory the skill sits in has swept +# the skill up with everything else, without ever spelling its name. +# +# Globs, as substrings: they are not words and can sit anywhere in a path. +_SWEEP_GLOBS = ("*", "?", "**", "rglob") + +# The rest are words, and matching them as substrings was the bug twice over. +# `-r ` is a flag an agent almost never types alone: `grep -ri`, `grep -rn` +# and `ls -lR` all sweep and none of them contains `-r ` -- so the shapes the +# docstring below names as the realistic ones were matched only in their +# tidiest spelling. The flag is now any single-dash cluster containing `r`, +# which is what those all are, and the commands are matched on a word +# boundary so a sweep is a command somebody ran rather than two letters +# inside another word. +# +# Bare `grep` is deliberately not here. `grep foo ` does not read the +# directory -- it is the recursive flag or a glob that does, and both are +# matched already -- so listing it would void verdicts on the strength of a +# command that never reached the skill. +# +# Lowercase only, deliberately: every caller passes `_normalise(...)`, so an +# upper-case spelling here can never match. `-R ` sat in the old tuple +# reading as case-sensitive coverage and was dead the day it was written; +# `\w*r\w*` covers it now because the text is lowercased first. +_SWEEP_WORDS = re.compile( + r"(?:^|\s)(?:-\w*r\w*|--recursive|find|ls|rg|tree|fd)(?=\s|$)" +) + + +# Tools whose whole job is to walk a tree. A call to one of these is a sweep +# whatever its arguments say, because the recursion is the tool rather than +# something the agent spelled out: `Grep(pattern="deploy", path=".claude")` +# reads every file under `.claude` and carries no wildcard, no `-r` and no +# path that names the skill. +# +# Matched on the function name, loosely, because the same two tools are +# spelled differently by different harnesses and an MCP server may wrap them +# under a name of its own. +_SEARCH_TOOLS = ("glob", "grep", "search") + + +def _searches(call) -> bool: + """Whether this call is a directory search by virtue of which tool it is.""" + name = _normalise(str(getattr(call, "function", "") or "")) + return any(tool in name for tool in _SEARCH_TOOLS) + + +def _swept(text: str, marks: tuple[str, ...]) -> bool: + """Whether this call read a directory containing the skill, wholesale. + + `find -name SKILL.md`, `grep -ri x /`, `rg x ` + and `cat /*/SKILL.md` all read the skill and none of them spells + its directory, so matching the skill's own path missed every one. An + agent that has been told nothing and is looking for something reaches for + exactly these, which made the miss likeliest on the baseline -- the one + side whose emptiness the whole command rests on. + + Reported as an unknown room rather than as contamination: the call could + sweep the skill and the record does not prove it did. Unknown and dirty + void the same verdicts, so the conservative answer costs nothing here. + """ + if not any(glob in text for glob in _SWEEP_GLOBS) and not _SWEEP_WORDS.search(text): + return False + return any(_mentions(text, ancestor) for ancestor in _ancestors(marks)) + + +def _ancestors(marks: tuple[str, ...]) -> set[str]: + """Every directory the skill sits under, by its own spelling. + + The whole chain, not just the immediate parent: `find ~/.claude -name + SKILL.md` reads the skill from two levels up and names neither the skill + nor the directory holding it. + + Stops short of the top level, which is where the cost of being wrong + turns over. A skill checked out under `/tmp/` would otherwise + contribute `/tmp` to this set, and then every `ls /tmp/...` in the run -- + including ones nowhere near the skill -- reads as a room nobody can + vouch for. So `/home/u` is kept and `/home` is not. A sweep of a + top-level directory does reach the skill, and flagging it would void + verdicts across whole runs on the strength of a wildcard. + + Where that top level is depends on the mark, and the two marks do not + have one in common. A single floor of three was calibrated for the + absolute mark and left the staged one -- `/`, three + segments on every registered harness -- with no ancestors at all, so + `ls .claude/skills/`, `find .claude -name SKILL.md` and `grep -r x + .claude/skills/` all came back as a clean room. That is the shape an + agent told nothing reaches for, on the one side whose emptiness the + whole command rests on. + + So the staged mark keeps every segment and the absolute one does not. + The floor is not about depth, it is about what else lives there: `/home` + holds every user on the machine and `/tmp` holds every run, while the + staged mark is relative to a workspace made for one sample and each of + its segments is the harness's own -- `.claude` holds skills and nothing + else of ours. A sweep of it reaches the skill and almost nothing a + baseline would touch for another reason, and the answer it buys is + `UNKNOWN_ROOM` rather than contamination, so the conservative reading + costs a verdict nobody could vouch for anyway. + """ + found: set[str] = set() + for mark in marks: + parts = mark.split("/") + # Absolute (`/x/...`, or a drive letter after `_normalise`) spends a + # segment on the empty string before the root, so three there and + # one here leave `/home/u` and `.claude` respectively. + floor = 3 if (parts[0] == "" or ":" in parts[0]) else 1 + while len(parts) > floor: + parts = parts[:-1] + if (parent := "/".join(parts)): + found.add(parent) + return found + + +# Said instead of a contamination reason when there is no way to look for one. +# Phrased as what is missing rather than as a finding: it is not a claim that +# the baseline was dirty, it is a refusal to claim it was clean. +UNKNOWN_ROOM = ( + "could not tell whether the baseline read the skill: the run did not " + "record where the skill was, and it is not on this machine" +) + + +def _skill_marks(skill: str) -> tuple[str, ...] | None: + """The paths that reading makes a room dirty, lowercased and separator-flat. + + Two of them. The skill's own directory, absolute, which is where an agent + that went looking would find it. And `/`, the staged copy + `no_sandbox.install_skill` writes under a workspace that changes per + sample, so only its tail is knowable here. The harness names that + directory -- `.claude/skills` for Claude Code, its own for anything else -- + so this asks it rather than spelling one harness's answer in, which would + read every other harness's staged copy as untouched. + + It asks the *default* harness, which is right while that is the only + harness this command will run on -- `staleness` refuses any other, and + `behavioral` on another harness stages its skill somewhere this would not + recognise. Worth revisiting the day that refusal lifts. + + The absolute mark is resolved here, against this process. That is right + for every caller there is: the check runs in the same process as the run + it is judging. It would be wrong for a reader rebuilding somebody else's + logs, which is why that reader will have to carry the path in the log + rather than work it out -- but no such reader exists yet, and a parameter + kept warm for one is a parameter nothing tests against reality. + + `None` when it cannot say where the skill was, which is an unknown room + rather than a clean one. Returning the staged mark alone looked like an + answer and was not: it is the subset of the check that cannot fail, so + every tool call that reached the skill by its real path came back clean + and the case was credited. + + The staged mark carries no leading separator on purpose. `no_sandbox` runs + the agent with `cwd=workspace`, which makes `cat .claude/skills/foo/...` + the natural shell form, and a mark anchored with a leading `/` missed + exactly that. `_mentions` supplies the boundary instead, which is what + keeps the mark from matching the middle of a longer name. + """ + try: + home = str(Path(config.active().skill_path(skill)).resolve()) + except (Exception, SystemExit): # noqa: BLE001 -- see below + # `skill_path` raises `SystemExit` rather than returning, so catching + # `Exception` alone let a skill the config cannot place take the + # whole command down. + return None + staged = f"{harnesses.get().skills_dir}/{skill}" + return (_normalise(home), _normalise(staged)) + + +# Characters that continue a path component. A mark followed or preceded by +# one of these is part of a longer name, not the name itself. +_NAME_CHARS = "-_." + + +def _mentions(text: str, mark: str) -> bool: + """Whether `text` names `mark` as a whole path component. + + Plain `in` was the bug twice over. It is what made `/{skill}/` match + `/run/user/1000`, and it survived in the shell branch after the structured + branch was fixed, where it read `.claude/skills/foo-extra` as skill `foo`'s + own files. A sibling skill whose name extends this one's is not this one. + + So a hit has to end at a boundary -- end of string, or a character that + cannot continue a component -- and start at one. `foo` matches in + `/x/.claude/skills/foo/SKILL.md` and in `cat .claude/skills/foo/S`, and + does not match in `foo-extra`, `foo.bak` or `myfoo`. + """ + width = len(mark) + start = 0 + while (at := text.find(mark, start)) >= 0: + start = at + 1 + after = text[at + width : at + width + 1] + if after and (after.isalnum() or after in _NAME_CHARS): + continue + before = text[at - 1 : at] if at else "" + if before and (before.isalnum() or before in _NAME_CHARS): + continue + return True + return False + + +def _strings(value) -> list[str]: + """Every string anywhere in a tool call's arguments. + + Flat rather than by field name. The first version of this checked a fixed + list of path-ish argument names, which is a closed set against an open tool + surface: it had no entry for Glob's `pattern`, which is often the only + argument a Glob call carries, so that call was invisible. MCP tools name + their arguments whatever they like, so the list would have kept drifting. + + Checking every string is safe here only because the marks are boundary + matched and absolute. That is what removed the false positives, not the + field list. + """ + if isinstance(value, str): + return [value] + if isinstance(value, dict): + return [s for v in value.values() for s in _strings(v)] + if isinstance(value, (list, tuple)): + return [s for v in value for s in _strings(v)] + return [] + + +def _read_the_skill(call, marks: tuple[str, ...], skill: str) -> str | None: + """Whether this call reached into the skill's own directory. + + Every string argument is tested as written and again with `~/` and + `$HOME/` written out, wherever in it they appear. The expansion before + this one fired only on a value that started with `~` and had no + whitespace -- a bare path -- so a `~` inside a Bash command line, which + is where an agent actually types one, was never expanded at all. + + `./x` and `../x` are deliberately *not* resolved against a cwd. The agent's + was `no_sandbox`'s per-sample workspace and appears nowhere in the + tool-call record, so `resolve()` here resolved them against skillscope's + own instead and made the verdict depend on which directory the report was + run from. (`_normalise` does collapse an *interior* `/../` or `/./`, + which needs no cwd to be right.) + + The graded side reads its own installed skill by design, so a hit there is + expected. It is the baseline where this means the room was not empty. + """ + arguments = getattr(call, "arguments", None) or {} + if not isinstance(arguments, dict): + return None + + reason = f"read the skill's own files ({skill})" + spellings: list[str] = [] + for value in _strings(arguments): + text = _normalise(value) + spellings.append(text) + if (expanded := _expand_home(text)) != text: + spellings.append(expanded) + + for text in spellings: + if any(_mentions(text, mark) for mark in marks): + return reason + + # The one mark that is not a path. `_bundled_skills` is the harness's own + # staging directory, so its name in any argument is the agent having found + # the staging area rather than a file of its own. + if any("_bundled_skills" in text for text in spellings): + return reason + + # Last, and weaker on purpose: a call that read a directory the skill + # sits in, wholesale. Answered after the exact matches above so a call + # that provably named the skill keeps the reason that says so. + # + # Asked of the call, not of each argument on its own. A Bash sweep is one + # string holding both halves -- the sweep word and the directory -- but + # Glob and Grep take them as separate arguments, `pattern` and `path`, + # and those are the tools an agent that has been told nothing reaches for + # first. Tested per argument, `{"pattern": "**/SKILL.md", "path": + # ""}` had a sweep word in one string and an ancestor in + # another and matched in neither, so the same sweep that was caught as a + # shell command came back as a clean room through the structured tool. + # A call to a search tool needs no sweep word at all: the recursion is + # the tool. `Grep` with `path` set to a directory walks everything under + # it, and `{"pattern": "deploy", "path": ".claude"}` carries no marker + # any string test could find -- the only evidence that this read the + # whole tree is which tool was called. + swept = _searches(call) or any( + glob in text for text in spellings for glob in _SWEEP_GLOBS + ) or any(_SWEEP_WORDS.search(text) for text in spellings) + if swept and any( + _mentions(text, ancestor) + for text in spellings + for ancestor in _ancestors(marks) + ): + return UNKNOWN_ROOM + return None + + +def _inspect_room( + routing_engine, sample, skill: str +) -> tuple[str | None, str | None]: + """`(the skill that fired, why the room was not what it claimed)`. + + One walk over the tool calls, answering both questions, because the second + is only meaningful beside the first. `detect_activation` reports a skill + nobody staged as `other:`, which is exactly the contamination this + looks for -- the runner's own `~/.claude/skills` are registered next to + the staged ones, and a behavioral run passes no `config_dir` to keep them + out. + + `fired` is `None` or exactly `skill`, never an `other:` marker. The two + questions are answered by separate branches for that reason: when they + shared one `if/elif`, a second `other:` hit arriving after `why` was + already set fell through to the `elif` and was recorded as the skill + having fired -- which reads downstream as "the skill activated" on exactly + the contaminated cases that most need the opposite. + + Reading the skill's own source is the second route, and the one isolation + only partly closes: `no_sandbox` runs the agent with `cwd=workspace`, but + nothing stops it naming an absolute path, so a directory the staging step + skipped is still on disk for it to find. The sandboxed `claude-code` leg + does close it, by isolating the filesystem. A baseline that read it was + not a baseline. + + Best effort: a shape this cannot walk is an unknown room, not a failed + case. + """ + fired: str | None = None + why: str | None = None + marks = _skill_marks(skill) + # Nowhere to check against, so the read half of the check cannot run. Said + # out loud rather than passed over: this lands the case `ungraded`, which + # is what an unanswerable question is worth. Applied after the walk, not + # before it, so a reason the walk *could* establish -- another skill + # firing, which needs no path at all -- is reported in preference to it. + unknown, marks = marks is None, marks or () + try: + for call in routing_engine._tool_calls(sample): + hit = routing_core.detect_activation( + routing_engine._activation_event(call), + [skill], + allow_body_path=routing_engine.ALLOW_BODY_PATH, + ) + if hit and hit.startswith("other:"): + if why is None: + why = f"another skill fired: {hit.removeprefix('other:')}" + elif hit and fired is None: + fired = hit + if why is None: + why = _read_the_skill(call, marks, skill) + except Exception: # noqa: BLE001 -- see the docstring + # A walk cut short inspected only the calls it reached, so evidence on + # any later call was never looked at. That is an unknown room whatever + # `unknown` says: it was computed before the walk, from whether there + # were marks to check against, and cannot speak for a walk that did + # not finish. Returning the completed-walk expression here would + # report a baseline nobody inspected as clean, which is the one + # direction this check must never fail in. + return fired, why or UNKNOWN_ROOM + return fired, why or (UNKNOWN_ROOM if unknown else None) + + +# Limits that stop an agent mid-answer rather than deciding anything about +# it. `operator` and `custom` are left out deliberately: one is somebody +# cancelling and the other is a task's own rule, and neither is the clock. +_CUT_SHORT = { + "time": "stopped at the sample's time limit", + "working": "stopped at the sample's working-time limit", + "message": "stopped at the message limit", + "token": "stopped at the token limit", +} + + +def _cut_short(sample) -> str | None: + """Why this sample was stopped before it finished, if it was. + + inspect records a limit as `sample.limit`, separately from `sample.error`, + so a run the clock ended arrives looking exactly like one the agent + finished and failed. Read defensively: `limit` is absent on older logs and + its shape is inspect's rather than ours. + """ + limit = getattr(sample, "limit", None) + if limit is None: + return None + kind = str(getattr(limit, "type", None) or limit).lower() + for name, reason in _CUT_SHORT.items(): + if name in kind: + return reason + return None + + def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: """Map one inspect `EvalLog` back onto skillscope's outcome objects. @@ -153,6 +618,10 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: for the same reason; the two now agree. Cases with no sample are still errored below, so nothing goes quietly missing. """ + # Imported here rather than at module scope: `engine/routing.py` imports + # this module, so a top-level import would close the cycle. + from . import routing as routing_engine # noqa: PLC0415 + prompts = {c.id: c.prompt for c in cases} outcomes: list[BehaviorOutcome] = [] @@ -170,8 +639,32 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: if sample.error is not None: error = f"{sample.error.message}" + elif (cut := _cut_short(sample)) is not None: + # An agent the clock stopped did not answer the question; it ran + # out of room to answer it. inspect reports that as `sample.limit` + # and leaves `error` unset, so the attempt arrived as an ordinary + # failure -- which on the baseline is the dangerous direction. The + # baseline runs second, on whatever budget the graded half left, + # and is the half that explores longest, so it is the one the + # limits cut. A case the skill never earned then reads `earns_it`, + # off a baseline that was interrupted rather than beaten. + error = cut elif not checks: error = "case has no behavioral assertions to grade" + elif ungraded := [c for c in checks if not scorers.graded(c)]: + # One check nobody could grade makes the whole attempt + # unanswerable rather than partly answered: a side passes only by + # meeting *every* positive claim, so a claim with no verdict + # leaves the result undecidable, not negative. Reported as an + # error so it lands in the errors column and gets triaged as + # infrastructure, which is what it is -- and so that + # `staleness.answered` drops the attempt instead of reading a + # grader that never answered as evidence about a skill. + error = "could not be graded: " + "; ".join( + c["detail"] for c in ungraded[:3] + ) + + activated, contaminated = _inspect_room(routing_engine, sample, skill) outcomes.append( BehaviorOutcome( @@ -183,6 +676,11 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: checks=checks, error=error, degraded=routing_core.is_provider_error(error), + activated=activated, + contaminated=contaminated, + # Which repeat this is, where `--attempts` asked for several. + # 1 on every sample of an ordinary single-epoch run. + attempt=int(getattr(sample, "epoch", 1) or 1), ) ) @@ -207,6 +705,7 @@ def _outcomes(log, skill: str, cases: list[Case]) -> list[BehaviorOutcome]: # stopped run reads back its own log and nobody else's. CALL_TOKEN = "skillscope_call" +# Which half of a staleness comparison a log is, and whose skill it is about. def stop_requested() -> bool: """Whether the deadline's graceful stage has asked the command to stop.""" @@ -292,8 +791,26 @@ def run( *, solver_factory, log_dir: str | None = None, + baseline: bool = False, + attempts: int = 1, + concurrency: int = 0, ) -> list[BehaviorOutcome]: - """Run every behavioral case, grouped by skill. Mirrors `behavior.run`.""" + """Run every behavioral case, grouped by skill. Mirrors `behavior.run`. + + `baseline` is passed through to the task and used in the progress lines. + It does not itself remove the skill: the caller's `solver_factory` does + that, and this says so in the output. + + `attempts` runs every case that many times, as inspect epochs, and the + epoch comes back on each sample as `BehaviorOutcome.attempt`. One attempt + is the ordinary run and leaves the logs exactly as they were. + + `concurrency` caps how many cases are in flight at once; 0 leaves + inspect's own default alone. Not raised by default: the agents these legs + drive hold real sandboxes and real rate limits, and a default that + multiplied either would be a change to every existing command rather than + an option on a slow one. + """ from inspect_ai import eval as inspect_eval sandbox_spec.require_provider() @@ -311,15 +828,31 @@ def run( outcomes.extend(_failed(skill, skill_cases, deadline.active().message())) continue - print(f"[behavioral] {skill}: {len(skill_cases)} case(s)", flush=True) + label = "baseline" if baseline else "behavioral" + each = "" if attempts <= 1 else f", {attempts} attempts each" + print(f"[{label}] {skill}: {len(skill_cases)} case(s){each}", flush=True) try: logs = run_eval( inspect_eval, - build_task(skill, skill_cases, model, solver_factory=solver_factory), + build_task( + skill, + skill_cases, + model, + solver_factory=solver_factory, + baseline=baseline, + ), model=model, model_args=models.model_args(model), log_dir=log_dir or str(Path(".skillscope") / "logs"), log_realtime=realtime_logging(), + # Repeats are inspect's epochs: one task, one log, each case + # sampled `attempts` times with the epoch on every sample. + # Running the leg twice over would have written two logs a + # side, which a reader pairing them cannot tell from two logs + # that collided -- so the repeats stay inside one log and the + # epoch on each sample is what tells them apart. + epochs=attempts, + **({"max_samples": concurrency} if concurrency else {}), # A sample that raises is that case's failure, not the batch's. # inspect's default ends the task on the first error, so one # container that would not start discarded every case already diff --git a/skillscope/engine/judge.py b/skillscope/engine/judge.py index f4f0a30..9046f7a 100644 --- a/skillscope/engine/judge.py +++ b/skillscope/engine/judge.py @@ -45,6 +45,24 @@ """ +class Ungradable(Exception): + """The judge could not reach a verdict. The run's failure, not the skill's. + + Raised rather than returned as `(False, reason)`. A `False` here is a + statement about what the agent did, and an infrastructure fault wearing + that shape is not a neutral mistake -- it is one that reliably produces + the most alarming reading available. `skillscope staleness` compares two + runs, so a baseline whose judge never answered reads as a case the skill + was needed for, and a graded run whose judge never answered reads as a + skill that did harm. Both are conclusions about a skill drawn from a + sandbox that would not list or a grader that returned prose. + + An exception rather than a third return value because the caller has to + do something different with it, and a `bool | None` invites being read as + falsy by every caller that does not know yet. + """ + + def is_image(path: str) -> bool: return PurePosixPath(path).suffix.lower() in IMAGE_SUFFIXES @@ -232,7 +250,7 @@ async def grade( # Say so rather than presenting an empty workspace as fact. A judge told # "no files" will confidently report the agent did nothing, which reads # as the skill failing when the sandbox is what failed. - return False, f"judge skipped: could not list the sandbox -- {exc}" + raise Ungradable(f"could not list the sandbox -- {exc}") from exc described, images = await artifacts(paths) @@ -277,7 +295,9 @@ async def grade( parsed = parse_verdict(output.completion or "") if parsed is None: - return False, f"judge gave no JSON verdict: {(output.completion or '')[:200]!r}" + raise Ungradable( + f"judge gave no JSON verdict: {(output.completion or '')[:200]!r}" + ) satisfied, reason = parsed return satisfied, f"judge: {reason}" diff --git a/skillscope/engine/no_sandbox.py b/skillscope/engine/no_sandbox.py index e614ede..b9d4cb0 100644 --- a/skillscope/engine/no_sandbox.py +++ b/skillscope/engine/no_sandbox.py @@ -417,6 +417,7 @@ def harness_solver( extra_flags: list[str] | None = None, stop_when_factory: Callable[[], Callable[[dict], str | None]] | None = None, room_names: list[str] | None = None, + install: bool = True, ): """Solver: install the skill, run the real CLI once, record what it did. @@ -453,6 +454,12 @@ def harness_solver( which skill fired, and everything after that is paid for and unread. Left `None`, this runs through `inspect_ai.util.subprocess` exactly as before. + `install` is what separates a graded run from its baseline. False stages + nothing, so the agent meets the prompt with no skill at all -- the failure + `install_skill` exists to prevent, asked for on purpose and labelled as + such by the caller. `skillscope staleness` runs both and compares them; + nothing else should pass False. + A factory rather than the rule itself because the solver is built once and run for every sample, while a budget counts *per case*. Handed a single rule, its tally would carry from one case into the next and the room would @@ -469,7 +476,8 @@ async def solve(state, generate): from inspect_ai.util import subprocess as sandbox_subprocess workspace = await _workspace() - install_skill(skill_dir, workspace, harness.skills_dir) + if install: + install_skill(skill_dir, workspace, harness.skills_dir) prompt = state.input_text spec = harnesses.RunSpec( @@ -548,6 +556,7 @@ def claude_code_no_sandbox( extra_flags: list[str] | None = None, stop_when_factory: Callable[[], Callable[[dict], str | None]] | None = None, room_names: list[str] | None = None, + install: bool = True, ): """`harness_solver` with Claude Code, the harness every caller has used.""" return harness_solver( @@ -559,4 +568,5 @@ def claude_code_no_sandbox( extra_flags, stop_when_factory, room_names, + install, ) diff --git a/skillscope/engine/scorers.py b/skillscope/engine/scorers.py index 1b3bb09..63e063e 100644 --- a/skillscope/engine/scorers.py +++ b/skillscope/engine/scorers.py @@ -22,15 +22,37 @@ CHECKS = "checks" -def _check(kind: str, expectation: str, passed: bool, detail: str = "") -> dict: +def _check( + kind: str, + expectation: str, + passed: bool, + detail: str = "", + graded: bool = True, +) -> dict: + """One expectation's result. `graded` False means nobody could tell. + + `passed` stays False where nothing graded it, so a reader that knows only + about pass and fail still refuses to call the case green. Anything + attributing a result to the skill has to read `graded` first. + + Always written rather than omitted when True, so the dict has one shape. + Readers still default it to True -- see `graded()` -- because a dict read + back from a log written before this existed does not carry the key. + """ return { "kind": kind, "expectation": expectation, "passed": passed, "detail": detail, + "graded": graded, } +def graded(check: dict) -> bool: + """Whether anybody actually graded this check. A missing key reads as True.""" + return bool(check.get("graded", True)) + + def searchable(state) -> str: """Everything in the run, for `logs_contain` to search. @@ -81,7 +103,13 @@ async def score(state, target) -> "Score": # would blame the agent for the harness's failure. for path in wanted: checks.append( - _check("files_exist", path, False, f"could not list the sandbox: {exc}") + _check( + "files_exist", + path, + False, + f"could not list the sandbox: {exc}", + graded=False, + ) ) files = None else: @@ -98,13 +126,23 @@ async def score(state, target) -> "Score": # Judged expectations last: the deterministic results are on screen # before the grader calls, which take a few seconds each, begin. - for statement in meta.get(convert.EXPECTED, []): - ok, reason = await judge.grade(statement, state, must_happen=True) - checks.append(_check("expected_behavior", statement, ok, reason)) - - for statement in meta.get(convert.UNEXPECTED, []): - ok, reason = await judge.grade(statement, state, must_happen=False) - checks.append(_check("unexpected_behavior", statement, ok, reason)) + for kind, key, must_happen in ( + ("expected_behavior", convert.EXPECTED, True), + ("unexpected_behavior", convert.UNEXPECTED, False), + ): + for statement in meta.get(key, []): + try: + ok, reason = await judge.grade( + statement, state, must_happen=must_happen + ) + except judge.Ungradable as exc: + # The polarity is why this cannot be left as a False. + # On `must_happen=False` a False asserts the forbidden + # thing *happened*, so a grader that never answered + # would be reported as the agent having done it. + checks.append(_check(kind, statement, False, str(exc), graded=False)) + continue + checks.append(_check(kind, statement, ok, reason)) passed = bool(checks) and all(c["passed"] for c in checks) return Score( diff --git a/skillscope/engine/verify.py b/skillscope/engine/verify.py index d06cde2..5d81935 100644 --- a/skillscope/engine/verify.py +++ b/skillscope/engine/verify.py @@ -218,8 +218,16 @@ def build_task( model: str, ctx: dict | None = None, effort: str | None = None, + install: bool = True, ): - """One task per skill, solved by real Claude Code rather than our agent.""" + """One task per skill, solved by real Claude Code rather than our agent. + + `install` False stages no skill, which is the baseline half of + `skillscope staleness`: the same cases, graded the same way, against an + agent that was never given the thing under test. The restore solver and + the agent's own skills argument have to agree about it, or the agent finds + a skill the report says it did not have. + """ from inspect_ai import Task from inspect_ai.solver import chain from inspect_swe import claude_code @@ -240,7 +248,8 @@ def build_task( bound = deadline.active() return Task( - name=f"claude-code-{skill}", + name=f"claude-code-{skill}" + ("" if install else "-baseline"), + tags=["skillscope", "graded" if install else "baseline"], setup=hooks.setup_solver(hook, skill, cases), cleanup=hooks.cleanup_fn(hook, skill, cases), dataset=samples, @@ -256,9 +265,11 @@ def build_task( # skill failed to do. solver=chain( _ensure_workdir(), - complete_skills_solver({skill_dir.name: skill_dir}), + complete_skills_solver({skill_dir.name: skill_dir} if install else {}), claude_code( - skills=[skill_dir], cwd=tools.workdir_path(), effort=effort or None + skills=[skill_dir] if install else [], + cwd=tools.workdir_path(), + effort=effort or None, ), ), scorer=scorers.expectations(), @@ -269,9 +280,23 @@ def build_task( def run( - skills: list[str], cases: list[Case], model: str, effort: str + skills: list[str], + cases: list[Case], + model: str, + effort: str, + *, + install: bool = True, + attempts: int = 1, + concurrency: int = 0, ) -> list[BehaviorOutcome]: - """Mirrors `behavior.run`, so the CLI and the benchmark treat it the same.""" + """Mirrors `behavior.run`, so the CLI and the benchmark treat it the same. + + `attempts` and `concurrency` are inspect's epochs and sample limit, as in + `behavioral.run`. Both legs take them, so a staleness run measures the + same way whichever engine it asked for -- an option that worked on one + engine and was silently dropped on the other would make two runs of the + same command incomparable without saying so. + """ from inspect_ai import eval as inspect_eval require() @@ -291,15 +316,22 @@ def run( ) continue - print(f"[claude-code] {skill}: {len(skill_cases)} case(s)", flush=True) + label = "claude-code" if install else "claude-code/baseline" + each = "" if attempts <= 1 else f", {attempts} attempts each" + print(f"[{label}] {skill}: {len(skill_cases)} case(s){each}", flush=True) logs = behavioral.run_eval( inspect_eval, - build_task(skill, skill_cases, model, effort=effort), + build_task( + skill, skill_cases, model, effort=effort, + install=install, + ), model=model, model_args=models.model_args(model), log_dir=str(Path(".skillscope") / "logs"), log_realtime=behavioral.realtime_logging(), display="plain", + epochs=attempts, + **({"max_samples": concurrency} if concurrency else {}), ) if not logs and behavioral.stop_requested(): # See `behavioral.run`. diff --git a/skillscope/staleness.py b/skillscope/staleness.py new file mode 100644 index 0000000..769c6c1 --- /dev/null +++ b/skillscope/staleness.py @@ -0,0 +1,1275 @@ +# Copyright Advanced Micro Devices, Inc. +# +# SPDX-License-Identifier: MIT + +"""Is the skill earning its place, or would the agent have done as well alone? + +A behavioral run says whether a skill's cases pass. It cannot say whether they +pass *because of the skill*, and those are different questions: a case can be +written so that any competent agent satisfies it, at which point a green report +is measuring the model rather than the thing under test. + +So the same cases are run twice -- once with the skill installed and once with +nothing installed at all -- and the two are placed side by side. `install_skill` +calls the second situation out as a failure mode, "answering from the prompt and +scoring like it, which looks like a bad skill rather than a missing one". Asked +for deliberately and labelled, that failure is the measurement. + +Three things a case can turn out to be, and an error state: + +``earns_it`` + Passes with the skill, fails without it. The case discriminates, which is + the only outcome that tells you the skill did something. Whether the agent + actually opened the skill is reported beside it, because installing a + skill does not make an agent use one and a case that moved without it + firing moved for a reason nobody has identified. +``harmful`` + Fails with the skill, passes without it. Rare, and worth stopping for: the + skill is actively steering the agent wrong. +``no_signal`` + Came out the same both ways. Two findings sharing a name, and the pass + counts beside the verdict say which you are holding. Both sides passing is + a case any agent satisfies, which is not evidence about the skill. Both + sides failing is a broken case or a genuinely hard one, which is not + evidence about the skill either, and points somewhere else entirely. +``ungraded`` + One side or both could not be graded: a side that errored throughout, a + grader that never answered, or a baseline that turned out not to be the + empty room it claimed to be. + +Three rather than more. Earlier versions split out whether the skill fired, +whether a case asserted anything positive at all, and whether a shared failure +was a broken case or a hard one. Each of those is real and each is still +reported -- as counts and notes beside the verdict, where the evidence for +them already was. None of them was a different verdict: each was one of these +verdicts with a caveat attached, and a caveat reads better as a sentence than +as a row in a table nobody can hold seven of. + +Each side can be run more than once (``--attempts``). A side counts as having +passed when any of its attempts met every positive claim, which at one attempt +each is exactly the rule above. "Any" rather than "all" because the question a +side answers is what the agent *can* do: an agent that managed the case once +managed it, and a side that failed the other two times has still shown the +task to be within reach of whatever it was holding. + +That makes more attempts a higher bar rather than a steadier reading of the +same one. ``earns_it`` at three attempts needs the baseline to have failed +three times out of three, which is a stronger claim than failing once; and a +baseline that passes once in three takes the case to ``no_signal`` where one +attempt each would have called it ``earns_it``. So a verdict is comparable +with another taken at the same attempt count and not with one taken at a +different count, and raising ``--attempts`` is asking a harder question rather +than asking the same one more carefully. + +The counts are carried into the report either way, because `3/3` against `0/3` +and `1/1` against `0/1` are the same verdict and not the same evidence, and +`3/3` against `1/3` is a third thing again. + +No verdict here gates, though an expired deadline or a refused run still exits +non-zero: those are statements about the run rather than about a skill. A +nondeterministic agent disagrees with itself between runs often enough that a +single pair is evidence rather than proof -- the same reason +`tools/benchmark_engines.py` measures engine agreement against how much one +engine already disagrees with itself, and the same reason the `claude-code` leg +is documented as reporting rather than gating. A number here is a question to +go and look into, not a build failure. Repeats are what let a reader weigh one; +they are not a licence for this command to start failing builds. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field + +from .behavior import BehaviorOutcome + +# (key, what it means) -- one row per verdict, in the order a reader should +# care about them. +VERDICTS: tuple[tuple[str, str], ...] = ( + ("earns_it", "passed with the skill, failed without it"), + ("harmful", "failed with the skill, passed without it"), + ("no_signal", "came out the same both ways"), + ("ungraded", "one side or both could not be graded"), +) + +# What a case claims the skill will *do*. Only these decide a verdict. +POSITIVE_KINDS = ("expected_behavior", "logs_contain", "files_exist") + +# What a case forbids. Deliberately excluded from the verdict. +# +# A guard rail exists to catch a future edit to the skill that starts doing the +# forbidden thing, so a well-written one holds whether or not the skill is +# installed -- the base model was never going to run `sudo` unprompted either. +# Counting that as "the skill made no difference" would mark the checks most +# worth keeping as dead weight, and push an author to delete them. They are +# reported on their own terms instead: held, or broken. +GUARD_KIND = "unexpected_behavior" + + +@dataclass +class Comparison: + """One case, run both ways, each way once or several times. + + No `diverged` flag is computed from the two sides' checks. The details are + judge prose, and two runs can word the same behaviour differently; a + boolean over that would claim a difference the harness cannot stand + behind. They are shown instead, for a reader to judge. + """ + + id: str + skill: str + verdict: str + # Attempts that produced a grade, and how many of them met every positive + # claim. One attempt each is the default; `passes_with` is then 0 or 1 and + # the verdict rules read exactly as they did before repeats existed. + attempts_with: int = 0 + attempts_without: int = 0 + passes_with: int = 0 + passes_without: int = 0 + # Attempts that errored instead. Counted apart from the graded ones so a + # side that half ran is not reported as a side that half failed: the + # denominator a reader sees is the number of attempts that answered. + errored_with: int = 0 + errored_without: int = 0 + error: str | None = None + # The checks of the attempt the verdict rests on -- one that passed, where + # the side has one, and otherwise the first. Both sides are carried rather + # than reduced to the verdict. A case that failed both ways can still have + # failed them *differently* -- the first run of this command had an agent + # that tried the task and was stopped by the machine on one side, and one + # that never attempted it on the other, which is the skill working. The + # verdict could not see it and neither could the JSON, so the only way to + # find out was to open the raw inspect logs, which is not a report. + checks_with: list[dict] = field(default_factory=list) + checks_without: list[dict] = field(default_factory=list) + # Every guard rail the installed side was held to, one row per guard, + # failed if *any* attempt broke it. + # + # Kept apart from `checks_with` because that is one attempt's evidence -- + # `_decisive` picks it on positive claims and firing, knowing nothing + # about guards -- and a guard rail is a claim about the whole side. With + # `--attempts` above one, reading guards off the decisive attempt meant a + # skill that ran the forbidden thing on an attempt that was not chosen + # was reported as having broken nothing at all. + guards_with: list[dict] = field(default_factory=list) + # The same, for the baseline. Carried because a guard the *baseline* + # broke and the installed side held is the skill preventing harm, which + # is a kind of value the pass/fail comparison cannot see at all: both + # sides can reach the same answer and only one of them wreck something on + # the way. The first real run of this command produced one -- an unaided + # agent attempting a sudo write to `/opt/rocm/share` on a case the skill + # kept read-only -- and the report credited the skill with nothing. + guards_without: list[dict] = field(default_factory=list) + # Whether the agent actually reached for the skill on an attempt the + # installed side passed. See `_fired_on_pass`: installing a skill does not + # make an agent use one, so an `earns_it` where this is False is a case + # that moved for a reason nobody has identified. A boolean rather than the + # skill's name, because the name was only ever this skill's own -- the + # room check reports either `None` or exactly the skill under test -- so + # it carried nothing the row did not already say. + fired_with: bool = False + # Why the baseline was not the empty room it claims to be, if it was not. + # Reported whenever it happened, on any attempt. + baseline_contaminated: str | None = None + # Whether that happened on a baseline attempt that *passed*. This is the + # one that voids a verdict, and the distinction only has teeth once a side + # can be attempted more than once -- a baseline that was dirty on an + # attempt it failed cannot have reached a conclusion it never reached. + baseline_dirty_on_pass: bool = False + # Wall seconds each side spent on this case, over the attempts that + # answered. `None` where the side has no honest number -- see `_seconds`, + # where the distinction between "no time" and "no measurement" is the + # difference between reporting the finding and inverting it. + seconds_with: float | None = None + seconds_without: float | None = None + + +def positive(checks: list[dict]) -> list[dict]: + """The checks that claim the skill will do something.""" + return [c for c in checks if c.get("kind") in POSITIVE_KINDS] + + +def guards(checks: list[dict]) -> list[dict]: + """The checks that forbid something.""" + return [c for c in checks if c.get("kind") == GUARD_KIND] + + +def _met(outcome: BehaviorOutcome) -> bool: + """Whether this attempt met every positive claim on the case.""" + claims = positive(outcome.checks) + return bool(claims) and all(c["passed"] for c in claims) + + +def answered(attempts: list[BehaviorOutcome]) -> list[BehaviorOutcome]: + """The attempts that answered, as opposed to the ones that errored. + + An attempt that errored has measured nothing. Keeping it in the + denominator would report a provider that dropped the connection as a side + that failed, which is the most flattering possible reading of an error on + the graded side and the least flattering on the baseline. + + A grader that could not reach a verdict sets `error` too, from + `engine/behavioral._outcomes`. That is the same category and the one most + likely to be mistaken for a result: a sandbox that would not list, or a + judge that answered in prose, used to arrive here as a check that simply + failed, which manufactured `earns_it` out of a broken baseline and + `harmful` out of a broken graded run. + """ + return [a for a in attempts if not a.error] + + +def _seconds(attempts: list[BehaviorOutcome]) -> float | None: + """Wall seconds this side spent on the case, over the attempts that answered. + + Summed rather than averaged or taken from the decisive attempt. The figure + has to reconcile with the per-half total in the table above it, and a mean + of medians adds up to nothing a reader can check. `_decisive` would be + worse still: it prefers an attempt that *passed*, which selects on exactly + the dimension that correlates with speed, so a baseline's one lucky fast + pass would be reported as its time while three slow failures vanished. At + one attempt a side -- the default -- the sum is that attempt's time. + + The denominator is already on the row: `_passes` prints `3/3 with, 0/3 + without` in the next column, so a reader seeing `7m12s` beside `3/3` reads + it as three attempts' worth without being told. + + `None`, not `0.0`, where no attempt produced a number. This is the one + column where reporting nothing as zero inverts the finding: the case this + command exists for is a baseline that ran for half an hour and never + finished, and `0s` beside the graded half's `2m23s` reads as the baseline + being instant. Zero is discarded for the same reason -- it is what a + synthesised failure carries for a case inspect never ran, and an agent run + is never genuinely instantaneous. + """ + timed = [a.elapsed_s for a in answered(attempts) if a.elapsed_s] + return round(sum(timed), 1) if timed else None + + +def _decisive(attempts: list[BehaviorOutcome]) -> BehaviorOutcome | None: + """The attempt a side's verdict rests on: one that passed, else the first. + + A side is credited with passing when *any* of its attempts did, so the + passing attempt is the one whose evidence belongs in the report. Where + nothing passed, every attempt is equally the answer and the first will do. + + Among passing attempts, one that reached for the skill is preferred over + one that did not. The verdict no longer turns on firing, but the note + beside it does, and the failure the preference prevents is unchanged: + `_fired_on_pass` asks whether the skill fired on any attempt that passed, + and this is what keeps `fired_with` and `checks_with` pointing at the same + attempt as that answer. A report saying a case earned its place while + showing the checks of an attempt where nothing fired would be describing a + different run from the one it graded. + + A side where every attempt errored falls back to the first of those. Its + verdict is `ungraded` whatever this returns, and an errored attempt can + still carry the checks it had graded before it stopped -- which is the + difference between a report that shows how far the side got and one that + shows a blank. + """ + scored = answered(attempts) + for attempt in scored: + if _met(attempt) and attempt.activated: + return attempt + for attempt in scored: + if _met(attempt): + return attempt + if scored: + return scored[0] + return attempts[0] if attempts else None + + +def _worst_guards(attempts: list[BehaviorOutcome]) -> list[dict]: + """One row per guard rail this side was held to, at its worst showing. + + A guard rail that held twice and broke once broke: it exists to catch the + skill doing the forbidden thing, and it caught it. So the rows are keyed + on the expectation and a failure replaces a pass, which also keeps the + count at one per guard however many attempts ran. + + Errored attempts are left out, as everywhere else: an attempt that + measured nothing cannot have seen a guard hold either. + """ + worst: dict[str, dict] = {} + for attempt in answered(attempts): + for guard in guards(attempt.checks): + seen = worst.get(guard["expectation"]) + if seen is None or (seen["passed"] and not guard["passed"]): + worst[guard["expectation"]] = guard + return list(worst.values()) + + +def _harm_prevented(case: dict) -> list[dict]: + """Guards the baseline broke that the installed side held. + + The skill stopped the agent doing something the case forbids. Asked per + expectation rather than per case, because a side can hold one guard and + break another, and it is the specific forbidden thing that a reader needs + named. + + A guard missing from the baseline is not counted. The baseline has to have + been *held to* the guard and failed it; a case the baseline never reached + tells us nothing about what it would have done. + """ + broke = { + g["expectation"]: g for g in case["guards_without"] if not g["passed"] + } + return [ + broke[g["expectation"]] + for g in case["guards_with"] + if g["passed"] and g["expectation"] in broke + ] + + +def classify( + with_skill: list[BehaviorOutcome], without: list[BehaviorOutcome] +) -> str: + """Which of `VERDICTS` this case's two sides are. + + Three verdicts and an error state, because three is what a pair of runs + can establish: the two sides differed in the skill's favour, they differed + against it, or they did not differ. + + Everything the longer taxonomy split out of those is still reported, as + counts and notes beside the verdict rather than as verdicts of their own. + Whether the skill fired is `fired_with`, and a case that moved without it + firing is flagged where it is listed. Whether a case asserts anything the + skill could be credited for is visible in its claims. Whether a shared + failure is a broken case or a hard one is the pass counts, which are + printed next to every row. None of those was ever a different *verdict*: + each was this verdict with a caveat attached, and a caveat reads better as + a sentence than as a row in a table nobody can hold seven of. + + Both arguments are a side's attempts: one each is the ordinary run, and + `--attempts` makes them longer. A side passed when any attempt met every + positive claim -- see `GUARD_KIND` for why the forbidden ones are kept out + of it, and the module docstring for why "any" rather than "all". + + A side with nothing left to grade is `ungraded` rather than a failure. One + attempt that errored out of three is not that case: the other two answered. + """ + here_scored, there_scored = answered(with_skill), answered(without) + if not here_scored or not there_scored: + return "ungraded" + here = any(_met(a) for a in here_scored) + there = any(_met(a) for a in there_scored) + if here and not there: + return _voided_if_dirty("earns_it", without) + if there and not here: + return _voided_if_dirty("harmful", without) + return _voided_if_dirty("no_signal", without) + + +def _fired_on_pass(attempts: list[BehaviorOutcome]) -> bool: + """Whether the skill fired on an attempt the installed side passed. + + Installing a skill does not make an agent use one, so a case that moved + without the skill firing has moved for some reason nobody has identified. + That is no longer a verdict of its own, but it is still the difference + between a case that credits the skill and one that merely coincides with + it, so it is carried on the comparison and flagged in the report. + + Asked over the whole side rather than over one attempt. Reading + `activated` off the attempt `_decisive` picked made the answer turn on + which of two passing attempts ran first: the same pair of runs in the + other order credited the skill, and the order two attempts happened to run + in is not a property of the skill. + """ + return any(_met(a) and a.activated for a in answered(attempts)) + + +# Verdicts a contaminated baseline cannot support, and why it cannot. +# +# Contamination is not symmetric. `harmful` rests on the baseline having +# *managed* something, and a baseline that reached another skill or read this +# skill's own files did not manage it unaided, which is the whole of what the +# verdict claims. +# +# `earns_it` rests on the baseline having *failed*. A baseline that failed +# while holding more than it should have fails all the harder without it, so +# the finding survives its own contamination and voiding it would throw away a +# conclusion the run is entitled to. +# +# `no_signal` is now two findings under one name, and the rule covers both +# without a branch. Both sides passing rests on the baseline having passed, so +# contamination voids it. Both sides failing rests on a baseline that passed +# nothing -- and `dirty_on_pass` only fires on an attempt that passed, so it +# is False there and the verdict stands. Merging the old `fails_either_way` +# and `guards_only` in here therefore voids nothing it should not. +VOIDED_BY_CONTAMINATION = ("no_signal", "harmful") + + +def dirty_on_pass(without: list[BehaviorOutcome]) -> bool: + """Whether a baseline attempt that *passed* was contaminated. + + Contamination on an attempt the baseline failed voids nothing: the verdicts + it voids are the ones claiming the baseline managed something, and an + attempt that did not manage it cannot be the attempt they rest on. With one + attempt a side this is the same rule as before -- `no_signal` and `harmful` + both require the baseline to have passed -- and it only parts company with + it once a side can pass on one attempt and be dirty on another. + """ + return any(a.contaminated and _met(a) for a in answered(without)) + + +def _voided_if_dirty(verdict: str, without: list[BehaviorOutcome]) -> str: + if verdict in VOIDED_BY_CONTAMINATION and dirty_on_pass(without): + return "ungraded" + return verdict + + +def _by_case(outcomes: list[BehaviorOutcome]) -> dict[str, list[BehaviorOutcome]]: + """One side's attempts, grouped by the case each answers. + + A dict of lists rather than `{o.id: o}`: with `--attempts` above one a case + has several attempts on a side, and keying on the id alone kept whichever + came last and silently threw the rest away. + """ + grouped: dict[str, list[BehaviorOutcome]] = {} + for outcome in outcomes: + grouped.setdefault(outcome.id, []).append(outcome) + return grouped + + +def _why_dirty(attempts: list[BehaviorOutcome]) -> str | None: + """Why the baseline was not the empty room it claimed, over its attempts. + + A known reason beats `UNKNOWN_ROOM`. They are different findings -- one + says the baseline demonstrably held something, the other that nobody could + tell -- and a side with one attempt of each has the answer to the second + in the first, so reporting "could not be checked" would be throwing it + away. Errored attempts are read too: a run that stopped partway still saw + what it saw before it stopped. + """ + from .engine.behavioral import UNKNOWN_ROOM + + reasons = [a.contaminated for a in attempts if a.contaminated] + known = [r for r in reasons if r != UNKNOWN_ROOM] + return known[0] if known else (reasons[0] if reasons else None) + + +def _counts(attempts: list[BehaviorOutcome]) -> tuple[int, int, int]: + """`(graded, passed, errored)` for one side of one case.""" + scored = answered(attempts) + return ( + len(scored), + sum(1 for a in scored if _met(a)), + len(attempts) - len(scored), + ) + + +def pair( + with_skill: list[BehaviorOutcome], without: list[BehaviorOutcome] +) -> list[Comparison]: + """Match the two runs up by case id. + + A case missing from either side is reported as `ungraded` rather than + dropped. The two runs are separate evals and either can lose a case, so + quietly comparing the intersection would shrink the denominator and make + every remaining number look more confident than it is. + """ + with_by_id = _by_case(with_skill) + without_by_id = _by_case(without) + comparisons: list[Comparison] = [] + + for case_id in sorted(set(with_by_id) | set(without_by_id)): + these = with_by_id.get(case_id, []) + those = without_by_id.get(case_id, []) + here, there = _decisive(these), _decisive(those) + with_n, with_passes, with_errors = _counts(these) + without_n, without_passes, without_errors = _counts(those) + missing = ( + "the baseline run" if not those + else "the graded run" if not these + else None + ) + comparisons.append( + Comparison( + id=case_id, + skill=(these or those)[0].skill, + # `classify` answers `ungraded` for a side with no attempt + # that answered, which a side that is absent entirely also + # is. Left to it rather than special-cased here, so one + # function decides what `ungraded` means. `missing` still + # earns its keep below, where it says *which* side was gone. + verdict=classify(these, those), + attempts_with=with_n, + attempts_without=without_n, + passes_with=with_passes, + passes_without=without_passes, + errored_with=with_errors, + errored_without=without_errors, + error=( + f"no result from {missing}" if missing + else next( + (a.error for a in these + those if a.error), None + ) + ), + guards_with=_worst_guards(these), + guards_without=_worst_guards(those), + checks_with=list(here.checks) if here else [], + checks_without=list(there.checks) if there else [], + fired_with=_fired_on_pass(these), + baseline_contaminated=_why_dirty(those), + baseline_dirty_on_pass=dirty_on_pass(those), + seconds_with=_seconds(these), + seconds_without=_seconds(those), + ) + ) + return comparisons + + +def summarize(comparisons: list[Comparison], meta: dict) -> dict: + from .engine.behavioral import UNKNOWN_ROOM + + counts = {key: 0 for key, _ in VERDICTS} + for comparison in comparisons: + counts[comparison.verdict] += 1 + + per_skill: dict[str, dict] = {} + for skill in sorted({c.skill for c in comparisons}): + subset = [c for c in comparisons if c.skill == skill] + per_skill[skill] = { + "cases": len(subset), + **{key: sum(1 for c in subset if c.verdict == key) for key, _ in VERDICTS}, + } + + graded = [c for c in comparisons if c.verdict != "ungraded"] + + # Guard rails, counted separately. A broken one is the serious finding + # here: a skill that made the agent do the forbidden thing. + broken = [ + c + for c in comparisons + if any(not g["passed"] for g in c.guards_with) + ] + guard_checks = sum(len(c.guards_with) for c in comparisons) + # The other direction, and the one nothing reported. A guard the baseline + # broke and the skill held is the skill earning its place by what it + # stopped the agent doing, which no verdict here can express: both sides + # can reach the same answer with only one of them wrecking something on + # the way. + prevented = [c for c in comparisons if _harm_prevented(asdict(c))] + return { + "meta": meta, + "totals": { + "cases": len(comparisons), + "graded": len(graded), + **counts, + # Of the cases graded both ways, the share that moved when the + # skill was taken away. Every `earns_it`, including the ones where + # the skill never fired -- those used to be their own verdict and + # left out of this number, and are now counted in it and flagged + # beside it. `earns_it_unfired` is how a reader discounts it, and + # the report says so next to the figure rather than computing a + # second share, which would be the old split under another name. + "discriminating": ( + round(sum(1 for c in graded if c.verdict == "earns_it") / len(graded), 3) + if graded + else None + ), + # `earns_it` cases where the skill never fired on an attempt + # that passed. The outcome moved; the skill is not shown to be + # what moved it. + "earns_it_unfired": sum( + 1 for c in comparisons + if c.verdict == "earns_it" and not c.fired_with + ), + "guard_checks": guard_checks, + "harm_prevented": len(prevented), + "guards_broken_with_skill": len(broken), + # Counted apart, because they are different findings. One says + # the baseline was demonstrably not empty; the other says nobody + # could tell. Both void the same verdicts -- an unanswerable + # question is worth no more than a failed one -- but a report that + # totalled them together would claim contamination it never found. + "baseline_contaminated": sum( + 1 + for c in comparisons + if c.baseline_contaminated + and c.baseline_contaminated != UNKNOWN_ROOM + ), + "baseline_unknown": sum( + 1 + for c in comparisons + if c.baseline_contaminated == UNKNOWN_ROOM + ), + # Attempts that errored, over every case and both sides. A run + # that lost a third of its attempts to a flaky provider has + # measured a third less than its case count suggests, and nothing + # else in these totals says so. + "errored_attempts": sum( + c.errored_with + c.errored_without for c in comparisons + ), + }, + "per_skill": per_skill, + "cases": [asdict(c) for c in comparisons], + } + + +def _reasons(case: dict) -> list[tuple[str, str, str]]: + """Each expectation that failed on either side, with both sides' details. + + Matched on the expectation text, which is what both runs were graded + against. A check present on one side only still appears, so a run that + produced fewer checks than its partner cannot hide the difference. + """ + with_by_text = {c["expectation"]: c for c in case["checks_with"]} + without_by_text = {c["expectation"]: c for c in case["checks_without"]} + rows: list[tuple[str, str, str]] = [] + for text in sorted(set(with_by_text) | set(without_by_text)): + here, there = with_by_text.get(text), without_by_text.get(text) + if (here and here["passed"]) and (there and there["passed"]): + continue + rows.append( + ( + text, + _detail(here), + _detail(there), + ) + ) + return rows + + +def _unsettled(case: dict) -> bool: + """Whether either side passed some of its attempts and failed others. + + It matters most for `no_signal`, whose advice is to stop treating the case + as evidence about the skill. That is right when both sides managed it + every time and wrong when a side managed it once in three: a baseline that + passed once is enough to deny the case `earns_it` -- see the module + docstring on what "any attempt passed" costs -- and nowhere near enough to + call the case dead. Telling an author to delete it on that footing is the + user-facing harm in the rule, and this is what separates the two. + + Always False at one attempt a side, where a side either passed or did not. + """ + return any( + attempts and 0 < passes < attempts + for passes, attempts in ( + (case["passes_with"], case["attempts_with"]), + (case["passes_without"], case["attempts_without"]), + ) + ) + + +def _passes(case: dict) -> str: + """`with / without` as attempts passed out of attempts graded. + + The verdict is what the two sides did; this is how often they did it. At + one attempt a side it reads `1/1 with, 0/1 without` and carries no more + than the verdict already said. Above one attempt it is the whole point, + because the verdict cannot show it: `3/3` against `0/3` is `earns_it` on + a baseline that never managed it, `3/3` against `1/3` is `no_signal` on a + baseline that managed it once, and `2/3` against `1/3` is `no_signal` on + two sides that both disagreed with themselves. Only the first is a result, + and only these counts say which one a reader is holding. + """ + lost = case["errored_with"] + case["errored_without"] + tail = f" ({lost} attempt(s) errored)" if lost else "" + return ( + f"{case['passes_with']}/{case['attempts_with']} with, " + f"{case['passes_without']}/{case['attempts_without']} without{tail}" + ) + + +def _seconds_cell(case: dict, side: str) -> str: + """One side's time for one case: a figure, or why there is not one. + + Three states and three strings, because the distinction is the whole + point of the column. A side that errored throughout has no time to show + and did not take zero seconds trying -- it is the half-hour baseline that + never concluded, which is the finding rather than a gap in it. + """ + seconds = case[f"seconds_{side}"] + if seconds is not None: + return _duration(seconds) + if case[f"errored_{side}"]: + return "did not finish" + return "not run" + + +def _advice(case: dict) -> str: + """What to do about one case, where the verdict alone does not say. + + This is where the collapsed taxonomy is paid for. `no_signal` now covers + three situations that want three different actions, and the single + sentence it used to carry -- "the agent already did this unaided" -- is + false for two of them. A case that failed on both sides was never managed + by anybody, and telling its author the agent managed it unaided sends + them to delete a case that is merely broken. + + Driven off the pass counts, which `_passes` is already printing in the + column alongside, so a reader can check the advice against the evidence + for it rather than taking it on trust. + """ + if case["verdict"] == "harmful": + return "the skill steered the agent wrong here" + if case["verdict"] == "earns_it": + return ( + "something moved, but the agent never opened the skill -- read " + "the run before crediting it" + ) + if not positive(case["checks_with"] + case["checks_without"]): + # The old `guards_only` verdict, as advice. Asked before the pass + # counts, because a case claiming nothing positive cannot pass and + # would otherwise be told it is broken or hard -- which would send its + # author to fix a case that is working exactly as written. + return ( + "this case claims nothing the skill could be credited for; its " + "guard rails were still checked" + ) + if not case["passes_with"] and not case["passes_without"]: + return ( + "neither side managed it; a broken case or a hard one, and not " + "yet evidence about the skill" + ) + if _unsettled(case): + return ( + "the two sides did not settle; run it again before reading " + "anything into it" + ) + return "the agent already did this unaided; the case is not evidence about the skill" + + +def _cell(value) -> str: + """A figure, or `n/a` where there is none. Never a zero standing in for one. + + Same spelling as `tools/benchmark_engines.py`, which reports the same + columns for the same reason. + """ + return "n/a" if value is None else str(value) + + +def _duration(seconds: float | None) -> str: + """`2m23s`, because `143.2s` beside `1680.4s` does not compare at a glance.""" + if seconds is None: + return "n/a" + minutes, rest = divmod(int(round(seconds)), 60) + return f"{minutes}m{rest:02d}s" if minutes else f"{rest}s" + + +def _ratio(halves: list[dict]) -> str | None: + """How much longer the baseline took, where both halves have a number.""" + graded, baseline = halves[0]["wall_time_s"], halves[1]["wall_time_s"] + if not graded or not baseline: + return None + return f"{baseline / graded:.1f}x" + + +def _spend_lead(halves: list[dict]) -> list[str]: + """The sentence this command exists to print, above everything else. + + A skill's worth is not only whether the agent gets there. An agent holding + the skill goes straight to the work and an agent without it explores, and + the gap between those is a result in its own right -- one the verdicts + cannot express, because both halves can pass and the case still be worth + keeping. The first run of this command had a graded half finish in 2m23s + against a baseline still going half an hour later, and the report said + nothing about it. + + Four shapes, because the honest sentence differs. A baseline that was + stopped is a floor and is said as one -- but only where it ran at all. A + graded half slower than its baseline is a finding and is said plainly + rather than suppressed: a skill that costs the agent time is something + its author wants to know. + + The halves share one bound, so the graded half can spend all of it and + leave the baseline none: every baseline case then returns failed without + an agent ever starting. Both halves are marked unfinished, which is true, + and reading only the baseline's flag told the reader it "was still going + 0s in" -- a baseline that never ran, reported as the slow exploring one + this command exists to find. The graded half's flag is what tells the two + apart. + """ + graded, baseline = halves + if not graded["finished"]: + return [ + "> **The bound ran out before the baseline could start.** The " + f"graded half used the whole {_duration(graded['wall_time_s'])} " + "and was stopped partway, so there is no baseline to compare it " + "against -- the cases below are reported against a half that " + "never ran. Raise `--timeout`, or cut the run down with " + "`--max-cases`, and read the verdicts only after that.", + "", + ] + if not baseline["finished"]: + return [ + f"> **The baseline never finished.** The graded half took " + f"{_duration(graded['wall_time_s'])}; the baseline was still " + f"going {_duration(baseline['wall_time_s'])} in, when the bound " + "stopped it. That gap is itself a result: whatever these cases " + "ask for, an agent without the skill reaches it only by " + "exploring, if at all.", + "", + ] + times = _ratio(halves) + if times is None: + return [] + if baseline["wall_time_s"] >= graded["wall_time_s"]: + return [ + f"> **The baseline took {times} the graded half's wall time** -- " + f"{_duration(graded['wall_time_s'])} with the skill, " + f"{_duration(baseline['wall_time_s'])} without it.", + "", + ] + return [ + "> **The graded half was the slower of the two** -- " + f"{_duration(graded['wall_time_s'])} with the skill, " + f"{_duration(baseline['wall_time_s'])} without it. The skill is " + "costing the agent time; whether it buys anything for that is the " + "verdict table below.", + "", + ] + + +def _spend_caveats(halves: list[dict], meta: dict) -> list[str]: + """Which of these columns compare, and which do not. + + The cost one is not decoration. Dollars arrive only through the host + leg's final event, which a half the bound killed never reaches -- so the + half that spent *more* is the likelier of the two to report nothing, and + a reader who takes a blank for a zero reads the result exactly backwards. + """ + notes = [ + "> Both halves ran the same cases on the same model, one after the " + "other, under one `--timeout`: the graded half goes first, so an " + "overrun there eats the baseline's budget. Wall time is the clock, " + "and includes waiting for retries and rate limits." + ] + if any("cost_usd" not in half for half in halves): + notes.append( + "> **A missing cost is a missing number, not a zero.** The " + "inspect-backed engines report one only where the provider " + "supplies pricing, which a gateway generally does not and which " + "inspect's own registry does not supply for any model; " + "`SKILLSCOPE_MODEL_PRICING` is the way in. Worse for this " + "report: dollars arrive with a run's final event, which a half " + "the bound stopped never reaches -- so the half that spent more " + "is the likelier one to show `n/a`. Tokens and model calls are " + "the like-for-like columns." + ) + if meta.get("concurrency"): + notes.append( + f"> Up to {meta['concurrency']} cases ran at once on each half, " + "so wall time measures how long the run took rather than how " + "much agent time it bought. Model calls and tokens are unaffected." + ) + if any(not half["finished"] for half in halves): + stopped = ", ".join(h["label"] for h in halves if not h["finished"]) + notes.append( + f"> The `{stopped}` row is a floor: the bound stopped it, so it " + "spent at least this and would have spent more." + ) + return notes + + +def _spend_table(halves: list[dict]) -> list[str]: + naming = {"graded": "with the skill", "baseline": "nothing installed"} + rows = [ + "| Half | Wall time | Model calls | Tokens | Cost |", + "| --- | --- | --- | --- | --- |", + ] + for half in halves: + floor = "" if half["finished"] else "+" + rows.append( + f"| {naming.get(half['label'], half['label'])} " + f"| {_duration(half['wall_time_s'])}{floor} " + f"| {_cell(half.get('model_calls'))} " + f"| {_cell(half.get('total_tokens'))} " + f"| {_cell(half.get('cost_usd'))} |" + ) + return rows + + +def _detail(check: dict | None) -> str: + if check is None: + return "not graded this side" + if check["passed"]: + return "passed" + return (check["detail"] or "failed").replace("|", "\\|").replace("\n", " ") + + +def render_markdown(summary: dict) -> str: + totals = summary["totals"] + meta = summary["meta"] + share = totals["discriminating"] + headline = ( + f"**{totals['earns_it']}/{totals['graded']} graded cases needed the skill**" + if totals["graded"] + else "**Nothing was graded both ways**" + ) + + # Still conditional rather than assumed: `summarize` is given whatever + # meta its caller built, and a report that names a model it was not told + # about states a measurement that was never taken. + ran_on = ( + f" on `{meta['model']}` (effort `{meta['effort']}`)" + if meta.get("model") and meta.get("effort") + else "" + ) + # How many times each side was attempted, said plainly: `3/3` against + # `0/3` and `1/1` against `0/1` are the same verdict and not the same + # evidence, and a reader who does not know which they are holding cannot + # tell them apart. Read off the cases rather than the invocation's own + # `--attempts`, so the report describes the run rather than the request. + per_side = {c["attempts_with"] for c in summary["cases"]} | { + c["attempts_without"] for c in summary["cases"] + } + most = max(per_side - {0}, default=1) + # Zero is kept here where `most` discards it. A side with no attempts at + # all is a case that ran on one side only -- `pair`'s "missing from either + # side", or a deadline that landed mid-run -- and the sentence below used + # to say every case ran twice straight over the top of it. + least = min(per_side, default=1) + sides = ( + "twice: once with the skill installed, once with nothing installed" + if most <= 1 + else f"up to {most} times on each side, with the skill installed and " + "with nothing installed" + ) + ran = ( + f"Some cases ran on one side only; the rest ran {sides}." + if least == 0 + else f"Every case ran {sides}." + ) + lines = [ + "## Skill staleness", + "", + f"{headline}{ran_on}. {ran}", + "", + ] + if most > 1: + # Said where it cannot be missed, because the verdict's meaning moves + # with the attempt count and nothing else in the report says so. "Any + # attempt passed" makes a higher count a higher bar: see the module + # docstring. + lines += [ + "> **Verdicts from this run compare with other runs at " + f"{most} attempt(s), not with runs at a different count.** A side " + "counts as having passed when any of its attempts did, so more " + "attempts is a harder question: `earns_it` needs the baseline to " + "have failed every single time.", + "", + ] + if meta.get("max_cases"): + # A capped run is not a statement about the suite. Said in the one + # place a reader cannot miss it, because a partial pass that reads as + # a whole one is the failure a cap invites. + lines += [ + f"> **This run covered only the first {meta['max_cases']} case(s) " + "of each selected skill** (`--max-cases`). It says nothing about " + "the cases it did not run.", + "", + ] + # What each half spent. Placed above the verdicts because it is the + # cheaper question and often the more useful answer: a skill whose cases + # all pass either way can still be the difference between two minutes and + # half an hour, and a verdict table alone reports that as "no signal". + # + # Absent from a summary built by something that did not run two halves, + # so `summarize` stays usable for a caller that has only comparisons. + halves = meta.get("legs") or [] + if len(halves) == 2: + lines += _spend_lead(halves) + + # Nothing moved at all. The row `earns_it | 0` is true and says nothing, + # and this is the one result the command exists to surface, so it is said + # rather than left to be noticed in a table. + # + # Split on whether anything passed, because the same verdict tally means + # two different things. A suite where both sides passed everything has + # cases that no longer test the skill. A suite where both sides failed + # everything has cases that do not work, and telling its author the skill + # has stopped earning its place would point them at the wrong thing + # entirely. + moved = totals["earns_it"] + totals["harmful"] + # Graded cases only, on both of these. An `ungraded` case is one nobody + # could draw a conclusion from, so letting its counts choose the headline + # lets a case that was never graded speak for the ones that were: a suite + # whose only graded case failed both ways, beside an ungraded case whose + # graded half happened to pass, was told the skill had stopped earning + # its place while the row underneath said the case was simply broken. + settled = [c for c in summary["cases"] if c["verdict"] != "ungraded"] + anything_passed = any( + c["passes_with"] or c["passes_without"] for c in settled + ) + # A suite that claims nothing positive anywhere is a third case again, + # and the one the old `guards_only` verdict used to catch here. Its cases + # cannot pass, so the "nothing passed" wording below would call a working + # suite broken; and they cannot move, so the "stopped earning its place" + # wording would blame a skill for not affecting checks that never asked + # anything of it. + claimed = any( + positive(c["checks_with"] + c["checks_without"]) for c in settled + ) + if totals["graded"] and not moved and not claimed: + lines += [ + "> **Every case here only forbids things.** Their guard rails were " + "checked, and nothing was claimed that the skill could be " + "credited with, so this run says nothing about whether the skill " + "is earning its place. Add a case that asserts what it should " + "make the agent *do*.", + "", + ] + elif totals["graded"] and not moved and anything_passed: + lines += [ + "> **No case in this suite shows the skill doing anything.** Every " + "graded case came out the same with the skill and without it. " + "Either the skill has stopped earning its place, or its cases " + "never tested what it knows -- read the skill and ask whether any " + "case here would need it. Those are different problems with " + "different fixes.", + "", + ] + elif totals["graded"] and not moved: + lines += [ + "> **No case in this suite passed on either side.** That is a " + "statement about the cases, not about the skill: nothing here has " + "been shown to work well enough to then ask whether the skill is " + "why. Fix the cases before reading anything into the verdicts.", + "", + ] + + if len(halves) == 2: + lines += ["### What each half cost", "", *_spend_table(halves), ""] + lines += [*_spend_caveats(halves, meta), ""] + + lines += [ + "Reporting only. Agents disagree with themselves between runs, so treat " + "a surprising verdict as somewhere to look rather than as a result.", + "", + "| Verdict | Cases | Means |", + "| --- | --- | --- |", + ] + for key, meaning in VERDICTS: + lines.append(f"| `{key}` | {totals[key]} | {meaning} |") + + if share is not None: + lines += ["", f"Discriminating: {share:.0%} of the cases graded both ways."] + + if totals["errored_attempts"]: + # Said out loud rather than left in the JSON. An attempt that errored + # is one nobody paid for an answer from, and a reader comparing `2/2` + # against `3/3` on the next case has no way to know the first was + # asked the same question three times. + lines += [ + "", + f"{totals['errored_attempts']} attempt(s) errored and are out of " + "the counts below. A case whose every attempt errored on a side " + "is `ungraded`; one that lost some of them is graded on the rest, " + "over the smaller denominator the counts show.", + ] + + from .engine.behavioral import UNKNOWN_ROOM + + unknown = [ + c + for c in summary["cases"] + if c["baseline_contaminated"] == UNKNOWN_ROOM + ] + if unknown: + lines += [ + "", + f"> **{len(unknown)} case(s) could not be checked for a clean " + "baseline.** The run did not record where the skill was, and it " + "is not on this machine, so there is nothing to recognise a " + "tool call against. They are reported `ungraded` where the " + "verdict rested on the baseline having managed something. " + "Re-running them records it.", + ] + + dirty = [ + c + for c in summary["cases"] + if c["baseline_contaminated"] and c["baseline_contaminated"] != UNKNOWN_ROOM + ] + if dirty: + # Three groups, because each rests on something different. A verdict + # in `VOIDED_BY_CONTAMINATION` claims the baseline *managed* + # something, so contamination kills it; the others rest on the + # baseline having *failed*, and a baseline that failed while holding + # more than it should have fails all the harder clean. The third + # group is what `--attempts` opened up: dirty on an attempt it + # failed, clean on the one it passed, so the attempt the verdict + # rests on is not the contaminated one. Driven off the constant so + # the rule and the wording cannot drift apart. + voided = [c for c in dirty if c["verdict"] == "ungraded"] + standing = [c for c in dirty if c["verdict"] != "ungraded"] + stood = [c for c in standing if c["verdict"] not in VOIDED_BY_CONTAMINATION] + # The third group splits again on whether the baseline ever passed. + # `no_signal` is two findings under one name, and the both-sides- + # failed one reaches here too: `dirty_on_pass` fires only on an + # attempt that passed, so a baseline dirty on an attempt it failed + # is never voided and lands beside the cases that did pass clean. + # Told as one group, it informed a case whose baseline passed + # nothing at all that it had passed on a clean attempt. + # `passes_without` counts on the same basis `dirty_on_pass` reads -- + # `_met` over `answered` -- so zero there means there was no passing + # attempt for contamination to have landed on. + elsewhere = [ + c for c in standing + if c["verdict"] in VOIDED_BY_CONTAMINATION and c["passes_without"] + ] + never_passed = [ + c for c in standing + if c["verdict"] in VOIDED_BY_CONTAMINATION and not c["passes_without"] + ] + named = lambda group: ", ".join( # noqa: E731 + f"`{c['id']}` ({c['verdict']})" for c in group + ) + lines += ["", f"> **{len(dirty)} case(s) had a baseline that was not " + "empty**: " + "; ".join(sorted( + {c["baseline_contaminated"] for c in dirty})) + "."] + if voided: + lines.append( + f"> {len(voided)} of them are reported `ungraded`: a baseline " + "that reached another skill, or read this skill's own " + "files, cannot show what an agent manages unaided." + ) + if stood: + lines.append( + f"> {len(stood)} still stand ({named(stood)}): they rest on " + "the baseline *failing*, and it failed while holding more " + "than it should have." + ) + if elsewhere: + lines.append( + f"> {len(elsewhere)} rest on a clean attempt " + f"({named(elsewhere)}): the baseline passed on an attempt " + "that was clean and was dirty on one it failed. Weaker " + "evidence than a run with no contamination at all." + ) + if never_passed: + lines.append( + f"> {len(never_passed)} still stand ({named(never_passed)}): " + "the baseline passed nothing on any attempt, so no verdict " + "here rests on something it managed while holding more than " + "it should have." + ) + + unfired = totals["earns_it_unfired"] + if unfired: + lines += [ + "", + f"> **{unfired} of the {totals['earns_it']} `earns_it` case(s) " + "passed without the skill ever firing.** Installing a skill does " + "not make an agent use it, so for those the outcome differed " + "without the skill being shown to be the reason. They are counted " + "in the figure above and flagged here: read the run before " + "crediting the skill with them.", + ] + + saved = [ + (c["id"], g) for c in summary["cases"] for g in _harm_prevented(c) + ] + if saved: + # Said before the guard tally below, which only ever reported guards + # the *skill* broke. A reader who saw "none was broken with the skill + # installed" learned that the skill did no harm, and never that it + # stopped any. + lines += [ + "", + f"> **The skill stopped the agent breaking {len(saved)} guard " + "rail(s) that it broke without it**: " + + "; ".join( + f"`{case_id}` -- {g['expectation'][:80]}" for case_id, g in saved + ) + + ". Both sides may still reach the same answer; this is what the " + "skill prevented on the way, which no verdict in the table above " + "can express.", + ] + + if totals["guard_checks"]: + broken = totals["guards_broken_with_skill"] + lines += [ + "", + f"{totals['guard_checks']} guard rail(s) checked and kept out of " + "the verdict: they hold whether or not the skill is installed, " + "which is what they are for. " + + ( + f"**{broken} case(s) broke one with the skill installed** -- " + "that is the skill doing the forbidden thing." + if broken + else "None was broken with the skill installed." + ), + ] + + notable = [ + c for c in summary["cases"] + if c["verdict"] in ("no_signal", "harmful") + or (c["verdict"] == "earns_it" and not c["fired_with"]) + ] + lines += ["", "### Cases worth a look", ""] + if not totals["graded"] and not totals["guard_checks"]: + # Not the same sentence as "none of them was worth a look". A run with + # nothing graded has no graded cases to make either claim about, and + # saying every one of them came out well is the same bug commit + # `542cad9` fixed one aisle over. + lines.append( + "No case was graded both ways, so there is nothing to look at here." + ) + elif not totals["graded"]: + # A case whose guards were checked is not nothing, even where no + # verdict was reached. "Nothing to look at here", printed two + # paragraphs under "that is the skill doing the forbidden thing", + # is the strongest finding this command can produce being + # contradicted by the weakest sentence in it. This section is about + # the verdict table, so it says that instead. + lines.append( + "No case reached a verdict. The guard rails above are what this " + "run found." + ) + elif not notable: + lines.append( + "None. Every graded case needed the skill, and reached for it." + ) + else: + lines += [ + "| Case | Verdict | Passed | Time (with / without) | What to do |", + "| --- | --- | --- | --- | --- |", + ] + for case in notable: + lines.append( + f"| `{case['id']}` | {case['verdict']} | {_passes(case)} " + f"| {_seconds_cell(case, 'with')} / {_seconds_cell(case, 'without')} " + f"| {_advice(case)} |" + ) + + # Every claim that came out differently on the two sides -- whatever the + # verdict. This used to skip `earns_it` cases, on the reasoning that a + # skill which earned its keep needs no explaining, which had it exactly + # backwards: those are the only cases where the evidence lives, and the + # section printed "nothing to show" on a run whose whole point was that + # three cases moved. + detailed = [c for c in summary["cases"] if _reasons(c)] + lines += ["", "### What the skill changed", ""] + if not detailed: + # Two findings that used to share one sentence. "Every claim matched" + # is a statement about the skill; "no case was graded on both sides" + # is a statement about the run, and printing the first for the second + # tells a reader the run concluded something it never reached. + compared = any( + c["checks_with"] and c["checks_without"] for c in summary["cases"] + ) + lines.append( + "Nothing: every claim came out the same with the skill and " + "without it." + if compared + else "Nothing was compared: no case was graded on both sides." + ) + for case in detailed: + lines += [ + f"**`{case['id']}`** ({case['verdict']}) -- {_passes(case)}; " + f"{_seconds_cell(case, 'with')} with, " + f"{_seconds_cell(case, 'without')} without", + "", + "| Expectation | With the skill | Without it |", + "| --- | --- | --- |", + ] + for text, here, there in _reasons(case): + lines.append( + f"| {text[:70].replace('|', chr(92) + '|')} " + f"| {here[:90]} | {there[:90]} |" + ) + lines.append("") + return "\n".join(lines) + "\n" diff --git a/skillscope/usage.py b/skillscope/usage.py index aa70d7f..127a5ca 100644 --- a/skillscope/usage.py +++ b/skillscope/usage.py @@ -17,7 +17,9 @@ from __future__ import annotations -from dataclasses import dataclass +import time +from contextlib import contextmanager +from dataclasses import dataclass, field @dataclass @@ -73,6 +75,76 @@ def record( _current.cost_usd = (_current.cost_usd or 0.0) + float(cost_usd) +@dataclass +class Leg: + """What one half of a paired run spent, and how long it took. + + One object rather than two loose numbers, because both cover exactly the + same interval. That is the only thing that entitles them to share a row. + + `finished` is the caller's to set. A half the deadline stopped reports a + floor rather than a measurement, and a report that cannot tell the two + apart turns "the baseline was still going after half an hour" into "the + baseline took half an hour", which is a weaker claim than the truth. + """ + + label: str + usage: Usage = field(default_factory=Usage) + wall_time_s: float = 0.0 + finished: bool = True + + def as_meta(self) -> dict: + return { + "label": self.label, + "wall_time_s": round(self.wall_time_s, 1), + "finished": self.finished, + **self.usage.as_meta(), + } + + +@contextmanager +def leg(label: str): + """Account one half of a paired run on its own, then fold it into the run. + + The global is *swapped*, not snapshotted and subtracted. `snapshot()` + hands back `_current` itself rather than a copy, so the obvious idiom -- + hold a snapshot, subtract it afterwards -- subtracts the totals from + themselves and reports every half as having spent nothing. That failure + produces plausible zeros rather than an error, which is the kind that + survives review. + + Everything that records does so by name off this module at call time + (`stats.record_log`, `record_stream_event`), so the swap reaches them + without either having to know a leg exists. Nothing records after the + engine returns, and the host leg's stream events run on the event loop + rather than a worker thread, so a scope around the call captures exactly + that half's spend even with several cases in flight. + + Folded back on the way out, so `snapshot()` still answers for the whole + command: the `**usage.snapshot().as_meta()` that every report already + splices stays the sum of the halves, and no existing report changes. + """ + global _current + outer, inner = _current, Usage() + half = Leg(label=label, usage=inner) + _current = inner + started = time.monotonic() + try: + yield half + finally: + half.wall_time_s = time.monotonic() - started + _current = outer + # Through `record` rather than by hand, so one place knows how to add + # one `Usage` into another -- including the rule that an unknown cost + # stays unknown rather than becoming zero. + record( + input_tokens=inner.input_tokens, + output_tokens=inner.output_tokens, + cost_usd=inner.cost_usd, + calls=inner.calls, + ) + + def record_stream_event(event: dict) -> None: """Record what one `claude` stream-json event says the run has spent. diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 8fb988d..7a75f50 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -34,8 +34,9 @@ import time import types import unittest +from dataclasses import asdict import urllib.error -from pathlib import Path +from pathlib import Path, PurePosixPath from unittest import mock import skillscope @@ -50,6 +51,7 @@ engine as engine_module, references, routing, + staleness, structure, ) from skillscope import selection as select_module @@ -66,6 +68,7 @@ from skillscope.engine import no_sandbox as engine_no_sandbox from skillscope.engine import behavioral as engine_behavioral from skillscope.engine import tools as engine_tools +from skillscope.engine import convert as engine_convert from skillscope.engine import scorers as engine_scorers from skillscope.engine import stats as engine_stats from skillscope.engine import pricing as engine_pricing @@ -2391,7 +2394,12 @@ def test_the_dispatch_reads_that_set_rather_than_a_literal(self) -> None: # The bug was a branch list that fell behind the choices list, so # the source is what to assert: every engine named, and an `else` # that raises rather than quietly picking one. - source = inspect.getsource(cli.cmd_behavioral) + # + # Read from `_run_behavioral` rather than `cmd_behavioral`: the + # dispatch moved there when `staleness` began running the same cases + # with the skill withheld, so both commands reach an engine by one + # route. The property being guarded is unchanged. + source = inspect.getsource(cli._run_behavioral) for engine in cli.ENGINES: with self.subTest(engine=engine): self.assertIn(f'args.engine == "{engine}"', source) @@ -5074,6 +5082,193 @@ def test_no_jobs_means_inspect_decides(self) -> None: self.assertIsNone(self.seen.get("max_samples")) +class TestAJudgeThatNeverAnsweredIsNotASkillThatEarnedIt( + unittest.IsolatedAsyncioTestCase +): + """A grader that could not answer used to be reported as the agent failing. + + `judge.grade` returned `(False, "judge skipped: ...")` when the sandbox + would not list, and `(False, "judge gave no JSON verdict: ...")` when the + grader answered in prose. Both are statements about the run, and both + arrived as a check that simply failed. + + In `behavioral` that is merely a red case. In `staleness` it is the + headline: `answered()` keeps any attempt without an `error`, so a baseline + whose judge broke counted as a baseline that genuinely failed, which is + `earns_it`, and a graded run whose judge broke counted as a skill doing + harm. The two verdicts the command exists to produce were the two a + grader hiccup manufactured. + """ + + async def test_a_sandbox_that_will_not_list_is_not_a_verdict(self) -> None: + async def _boom(): + raise engine_tools.ListingFailed("no container") + + with mock.patch.object(engine_judge, "tools_list_paths", _boom): + with self.assertRaises(engine_judge.Ungradable): + await engine_judge.grade("it wrote a file", object(), must_happen=True) + + async def test_a_grader_answering_in_prose_is_not_a_verdict(self) -> None: + # The polarity is why this one matters most. On `must_happen=False` a + # `False` asserts the forbidden thing *happened*, so a grader that + # rambled used to be reported as the agent having done it. + async def _none(): + return [] + + class _Model: + async def generate(self, _messages): + return types.SimpleNamespace(completion="I could not decide.") + + with mock.patch.object(engine_judge, "tools_list_paths", _none), \ + mock.patch("inspect_ai.model.get_model", lambda *a, **k: _Model()): + with self.assertRaises(engine_judge.Ungradable): + await engine_judge.grade( + "it called a cloud API", _JudgeState(), must_happen=False + ) + + +class _JudgeState: + """The little of `TaskState` that `judge.grade` reads to build its evidence.""" + + messages: list = [] + output = None + + +class TestAnUngradedCheckIsAnErrorNotAFailure(unittest.TestCase): + """The attempt is unanswerable, not failed, and the difference is the point. + + A side passes only by meeting *every* positive claim, so one claim with no + verdict leaves the side undecidable rather than negative. Routed through + `error` because that is the channel `staleness.answered` already drops, and + because it puts the fault in the errors column where it gets triaged as + infrastructure instead of read as the skill violating an expectation. + """ + + @staticmethod + def _case(case_id: str): + return datasets.Case( + id=case_id, skill="alpha", prompt="p", skill_should_trigger=True, + ) + + @staticmethod + def _log(checks): + sample = types.SimpleNamespace( + id="c1", + scores={"s": types.SimpleNamespace( + metadata={engine_scorers.CHECKS: checks} + )}, + error=None, + total_time=1.0, + ) + return types.SimpleNamespace( + status="success", error=None, samples=[sample] + ) + + def test_a_check_nobody_graded_errors_the_attempt(self) -> None: + log = self._log([ + {"kind": "expected_behavior", "expectation": "e", "passed": False, + "detail": "could not list the sandbox -- no container", + "graded": False}, + ]) + outcome = engine_behavioral._outcomes(log, "alpha", [self._case("c1")])[0] + self.assertIn("could not be graded", outcome.error or "") + self.assertFalse(outcome.passed) + + def test_an_ordinary_failure_is_still_a_failure(self) -> None: + # The other direction, and the one a careless fix breaks: a case that + # genuinely failed its expectation must stay a failure, not become an + # error that `staleness` then declines to count at all. + log = self._log([ + {"kind": "expected_behavior", "expectation": "e", "passed": False, + "detail": "judge: it did not", "graded": True}, + ]) + outcome = engine_behavioral._outcomes(log, "alpha", [self._case("c1")])[0] + self.assertIsNone(outcome.error) + self.assertFalse(outcome.passed) + + def test_a_check_from_an_older_log_reads_as_graded(self) -> None: + # Logs written before `graded` existed have no such key, and must not + # all become ungraded the moment this lands. + self.assertTrue(engine_scorers.graded({"passed": True, "kind": "k"})) + log = self._log([{"kind": "k", "expectation": "e", "passed": True, + "detail": ""}]) + outcome = engine_behavioral._outcomes(log, "alpha", [self._case("c1")])[0] + self.assertIsNone(outcome.error) + self.assertTrue(outcome.passed) + + +class TestABrokenJudgeOnTheBaselineIsNotEvidenceAboutTheSkill(unittest.TestCase): + """The regression itself, stated as the verdict it used to manufacture. + + This is the one that would have shipped: a baseline attempt whose grader + never answered, paired against a graded attempt that passed, reported as + `earns_it` -- the skill credited for a case the baseline was never + actually asked. + """ + + # Both halves are built by running `_outcomes` over a log, not by handing + # `pair` an outcome with `error` already set. Setting it by hand tests + # that `classify` honours an error it was given, which was never the bug + # and passes with the fix reverted. The claim here is about the chain: an + # ungraded check has to *become* that error before `pair` ever sees it. + # + # The module-level `_outcome` helper is used where a plain outcome is + # wanted, rather than a method of this class: `self._outcome` is where + # `unittest.TestCase` keeps the running test's `_Outcome`, so a helper by + # that name is shadowed during the run while looking right on the class. + + @staticmethod + def _side(check: dict) -> list: + sample = types.SimpleNamespace( + id="c1", + scores={"s": types.SimpleNamespace( + metadata={engine_scorers.CHECKS: [check]} + )}, + error=None, + total_time=1.0, + ) + log = types.SimpleNamespace(status="success", error=None, samples=[sample]) + case = datasets.Case( + id="c1", skill="alpha", prompt="p", skill_should_trigger=True, + ) + return engine_behavioral._outcomes(log, "alpha", [case]) + + def test_a_baseline_nobody_could_grade_comes_out_ungraded(self) -> None: + with_skill = self._side( + {"kind": "expected_behavior", "expectation": "e", "passed": True, + "detail": "judge: it did", "graded": True} + ) + without = self._side( + {"kind": "expected_behavior", "expectation": "e", "passed": False, + "detail": "could not list the sandbox -- no container", + "graded": False} + ) + comparison = staleness.pair(with_skill, without)[0] + self.assertEqual( + comparison.verdict, "ungraded", + "a baseline whose judge never answered is not a baseline that failed", + ) + + def test_a_baseline_that_really_failed_is_still_graded(self) -> None: + # The control. Without it the fix could pass by calling everything + # ungraded, which would make the command useless rather than wrong. + # + # Asserted as "graded, and the skill is the side that passed" rather + # than as a verdict name: which name a case like this carries is what + # the taxonomy collapse is about, and this test is about the judge. + with_skill = self._side( + {"kind": "expected_behavior", "expectation": "e", "passed": True, + "detail": "judge: it did", "graded": True} + ) + without = self._side( + {"kind": "expected_behavior", "expectation": "e", "passed": False, + "detail": "judge: it did not", "graded": True} + ) + comparison = staleness.pair(with_skill, without)[0] + self.assertNotEqual(comparison.verdict, "ungraded") + self.assertEqual((comparison.passes_with, comparison.passes_without), (1, 0)) + + class TestOneBadSampleDoesNotFailTheWholeBatch(unittest.TestCase): """A behavioral batch discarded every case it had already graded. @@ -5980,5 +6175,2998 @@ def test_a_good_table_still_binds(self) -> None: self.assertTrue(engine_models.cost_limit_binds("mockllm/model")) +def _outcome( + case_id: str, + passed: bool, + *, + error: str | None = None, + skill: str = "s", + activated: str | None = "s", + attempt: int = 1, +): + """One behavioral outcome, as either half of a staleness pair would report it. + + Carries a positive claim matching `passed`, because that is what a verdict + is computed from -- an outcome asserting nothing is `guards_only`, which + has its own tests. + + `activated` defaults to the skill having fired, which is the ordinary run + and keeps every test that is not about activation about what it is about. + The tests that *are* about it pass `activated=None` -- see + `TestEarningItMeansTheSkillActuallyFired`, where a case that moved without + the skill firing is `unattributed` rather than `earns_it`. + """ + return behavior.BehaviorOutcome( + id=case_id, skill=skill, prompt="p", passed=passed, elapsed_s=0.0, error=error, + activated=activated, + attempt=attempt, + checks=[{ + "kind": "expected_behavior", "expectation": "do the thing", + "passed": passed, "detail": "", + }], + ) + + +class TestStalenessSeparatesTheSkillFromTheModel(unittest.TestCase): + """The four verdicts, and why each is not the others.""" + + def test_a_case_that_needs_the_skill_is_the_one_that_moved(self) -> None: + self.assertEqual( + staleness.classify([_outcome("a", True)], [_outcome("a", False)]), "earns_it" + ) + + def test_a_case_that_passes_unaided_is_not_evidence(self) -> None: + # The finding the command exists for: green with the skill, green + # without it, so the report was measuring the model. + self.assertEqual( + staleness.classify([_outcome("a", True)], [_outcome("a", True)]), "no_signal" + ) + + def test_a_skill_that_makes_things_worse_is_called_that(self) -> None: + self.assertEqual( + staleness.classify([_outcome("a", False)], [_outcome("a", True)]), "harmful" + ) + + def test_a_case_failing_both_ways_says_nothing_either(self) -> None: + # Both-ways-failed and both-ways-passed are one verdict now. They are + # the same answer to the question this command asks -- the sides did + # not differ -- and what separates them is what to do about it, which + # the pass counts and `_advice` carry. See `_advice`: this one must + # not be told the agent managed it unaided. + self.assertEqual( + staleness.classify([_outcome("a", False)], [_outcome("a", False)]), + "no_signal", + ) + case = staleness.pair([_outcome("a", False)], [_outcome("a", False)])[0] + self.assertIn("neither side managed it", staleness._advice(asdict(case))) + + def test_an_errored_side_is_ungraded_rather_than_no_difference(self) -> None: + # A dropped connection measured nothing. Reading it as "the skill made + # no difference" is the most flattering possible reading of an error. + self.assertEqual( + staleness.classify( + [_outcome("a", False, error="API Error: 504")], [_outcome("a", False)] + ), + "ungraded", + ) + + +class TestStalenessPairsEveryCaseItWasAskedFor(unittest.TestCase): + def test_a_case_missing_from_one_side_is_kept_as_ungraded(self) -> None: + # Comparing the intersection would shrink the denominator and make + # every surviving number look more confident than it is. + comparisons = staleness.pair([_outcome("a", True), _outcome("b", True)], [_outcome("a", False)]) + verdicts = {c.id: c.verdict for c in comparisons} + self.assertEqual(verdicts, {"a": "earns_it", "b": "ungraded"}) + missing = next(c for c in comparisons if c.id == "b") + self.assertIn("baseline", missing.error) + + def test_the_share_is_over_what_was_graded_both_ways(self) -> None: + comparisons = staleness.pair( + [_outcome("a", True), _outcome("b", True), _outcome("c", True)], + [_outcome("a", False), _outcome("b", True)], + ) + totals = staleness.summarize(comparisons, {"model": "m", "effort": "e"})["totals"] + self.assertEqual(totals["cases"], 3) + self.assertEqual(totals["graded"], 2) + self.assertEqual(totals["discriminating"], 0.5) + + def test_nothing_graded_leaves_the_share_unset_rather_than_zero(self) -> None: + # Zero would read as "the skill never mattered", which is a result. + # None is the absence of one. + comparisons = staleness.pair([_outcome("a", True)], []) + totals = staleness.summarize(comparisons, {"model": "m", "effort": "e"})["totals"] + self.assertIsNone(totals["discriminating"]) + + +class TestTheStalenessReportLeadsWithTheCaveat(unittest.TestCase): + def test_the_reader_is_told_it_does_not_gate_before_the_numbers(self) -> None: + comparisons = staleness.pair([_outcome("a", True)], [_outcome("a", True)]) + summary = staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + report = staleness.render_markdown(summary) + self.assertIn("Reporting only", report) + self.assertLess(report.index("Reporting only"), report.index("| Verdict |")) + + def test_a_case_that_needed_nothing_is_named_with_what_to_do(self) -> None: + comparisons = staleness.pair([_outcome("a", True)], [_outcome("a", True)]) + report = staleness.render_markdown( + staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + ) + self.assertIn("`a`", report) + self.assertIn("unaided", report) + + +@contextlib.contextmanager +def _staleness_harness(test, *, dataset=None): + """A repo `cmd_staleness` can be run against, with the two halves recorded. + + The command was never called by a test before this: every assertion about + it read its source. A regression that called `_run_behavioral` twice with + the same `install` -- which destroys the entire measurement, since the two + halves become the same run -- passed the whole suite. + + `_run_behavioral` is the seam, so what each half was asked for is the + thing recorded. Everything below it reaches a provider. + """ + repo = Repo(test) + repo.skill("alpha", dataset=dataset or _behavioral_dataset("alpha")) + repo.activate() + + calls: list[dict] = [] + + def record(args, skills, gradable, *, install=True, config_dir=None): + calls.append( + { + "install": install, + "config_dir": config_dir, + "cases": [c.id for c in gradable], + } + ) + # Spend something, and a different something per half, so a test can + # tell which half a figure came from. An engine records from inside + # this call, which is the whole reason the leg scope wraps it. + usage.record( + input_tokens=10 if install else 100, + output_tokens=1 if install else 20, + calls=1 if install else 7, + ) + # Outcomes that differ by half, so the verdict the command computes + # is one only the right pairing can produce. Returning `[]` from both + # halves -- which this did -- meant no verdict was ever compared, so + # `pair(with_skill, without)` could be called with its arguments + # swapped and every test stayed green, while every real report had + # `earns_it` and `harmful` the wrong way round. + return [ + behavior.BehaviorOutcome( + id=case.id, skill=case.skill, prompt=case.prompt, + passed=install, elapsed_s=1.0 if install else 9.0, + checks=[{"kind": "expected_behavior", "expectation": "e", + "passed": install, "detail": ""}], + activated="alpha" if install else None, + ) + for case in gradable + ] + + reports: list[dict] = [] + harness = types.SimpleNamespace(repo=repo, calls=calls, reports=reports) + + def run(engine="claude-code", extra=None): + argv = [ + "staleness", "--engine", engine, "--skip-preflight", + "--model", "mockllm/model", *(extra or []), + ] + args = cli.build_parser().parse_args(argv) + return cli.cmd_staleness(args) + + harness.run = run + + with mock.patch.object(cli, "_run_behavioral", record), \ + mock.patch.object( + cli, "_write_report", + lambda summary, *a, **k: reports.append(summary)), \ + mock.patch.object(cli, "_prepare_graded_run", lambda *a, **k: ["alpha"]): + usage.reset() + yield harness + + +def _behavioral_dataset(slug: str) -> dict: + """`tier0_dataset`, with every triggering case asserting a behaviour. + + `_gradable_cases` keeps only cases with something to grade, so a routing + dataset leaves `cmd_staleness` with nothing to run and it returns before + reaching either half. + """ + data = tier0_dataset(slug) + for evaluation in data[EVALUATIONS_KEY]: + if evaluation[TRIGGER_KEY]: + evaluation["expected_behavior"] = ["do the thing"] + return data + + +def _checked( + case_id: str, + passed: bool, + checks: list[dict], + skill: str = "s", + *, + activated: str | None = "s", + attempt: int = 1, +): + """A behavioral outcome carrying graded checks. + + `activated` defaults to the skill having fired, for the reason `_outcome` + gives. + """ + return behavior.BehaviorOutcome( + id=case_id, skill=skill, prompt="p", passed=passed, elapsed_s=0.0, + checks=checks, activated=activated, attempt=attempt, + ) + + +def _check(text: str, passed: bool, detail: str = ""): + return {"kind": "expected_behavior", "expectation": text, "passed": passed, "detail": detail} + + +class TestBothSidesReasonsSurviveIntoTheReport(unittest.TestCase): + """The first real run of this command failed both ways for *different* + reasons -- the agent tried the task and was blocked on one side, and never + attempted it on the other -- and neither the verdict nor the JSON could + show it. Only the raw inspect logs could, which is not a report.""" + + def _pair(self): + return staleness.pair( + [_checked("c", False, [ + _check("Install it", False, "tried, blocked by missing sudo"), + _check("Avoid the cloud", True), + ])], + [_checked("c", False, [ + _check("Install it", False, "only surveyed, never attempted"), + _check("Avoid the cloud", True), + ])], + ) + + def test_both_sides_checks_reach_the_comparison(self) -> None: + comparison = self._pair()[0] + self.assertEqual(len(comparison.checks_with), 2) + self.assertEqual(len(comparison.checks_without), 2) + + def test_an_expectation_both_sides_met_is_not_reported_as_a_reason(self) -> None: + # Only what went unmet needs explaining. + rows = staleness._reasons(asdict(self._pair()[0])) + self.assertEqual([text for text, _, _ in rows], ["Install it"]) + + def test_the_two_reasons_appear_side_by_side(self) -> None: + summary = staleness.summarize(self._pair(), {"model": "m", "effort": "e"}) + report = staleness.render_markdown(summary) + self.assertIn("blocked by missing sudo", report) + self.assertIn("only surveyed, never attempted", report) + + def test_a_clean_run_is_not_described_as_every_case_needing_the_skill(self) -> None: + # A `no_signal` case has no unmet expectation either, so the empty + # state is about unmet expectations rather than about the verdict. + # The first run with a no_signal case reported the opposite. + comparisons = staleness.pair( + [_checked("c", True, [_check("Do the thing", True)])], + [_checked("c", True, [_check("Do the thing", True)])], + ) + summary = staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + report = staleness.render_markdown(summary) + self.assertEqual(summary["totals"]["no_signal"], 1) + # The string the code emits is "Every graded case needed the skill"; + # the lower-cased paraphrase this used to assert the absence of + # appears nowhere, so that line could not fail. + self.assertNotIn("Every graded case needed the skill", report) + self.assertIn("every claim came out the same", report) + + def test_a_check_only_one_side_graded_is_still_shown(self) -> None: + # Fewer checks on one side must not hide the difference. + comparisons = staleness.pair( + [_checked("c", False, [_check("Install it", False, "blocked")])], + [_checked("c", False, [])], + ) + rows = staleness._reasons(asdict(comparisons[0])) + self.assertEqual(rows[0][2], "not graded this side") + + +class _StubSample: + def __init__(self, sample_id, checks, error=None, epoch=1): + self.id = sample_id + self.scores = {"expectations": types.SimpleNamespace(metadata={"checks": checks})} + self.error = error + self.total_time = 1.0 + # inspect stamps the repeat number on every sample. `--attempts` + # above one makes several samples share a case id, and this is what + # tells them apart. + self.epoch = epoch + + +class TestARepeatIsTakenFromTheSampleThatRanIt(unittest.TestCase): + """`attempt` has to come off the sample's own epoch, not a constant. + + Every other multi-attempt test builds `BehaviorOutcome`s by hand with + `attempt=i`, so the one line in production that derives the field -- + `attempt=int(getattr(sample, "epoch", 1) or 1)` -- could have been + hard-coded to 1 or read the wrong attribute with the suite still green. + That line is what keeps `--attempts` repeats of one case apart: collapse + it and `_by_case` has one attempt where the run had three, with the + others silently dropped. + """ + + @staticmethod + def _case(case_id: str): + return datasets.Case( + id=case_id, skill="alpha", prompt="p", skill_should_trigger=True, + ) + + @staticmethod + def _checks(): + return [{"kind": "expected_behavior", "expectation": "e", + "passed": True, "detail": "", "graded": True}] + + def test_each_repeat_carries_its_own_number(self) -> None: + log = types.SimpleNamespace( + status="success", + error=None, + samples=[_StubSample("c1", self._checks(), epoch=n) + for n in (1, 2, 3)], + ) + outcomes = engine_behavioral._outcomes(log, "alpha", [self._case("c1")]) + self.assertEqual([o.attempt for o in outcomes], [1, 2, 3]) + + def test_a_sample_with_no_epoch_is_the_first_attempt(self) -> None: + # Ordinary single-pass runs, and any log written before epochs were + # asked for, have to read as attempt 1 rather than 0 -- `_by_case` + # keys on it. + sample = _StubSample("c1", self._checks()) + del sample.epoch + log = types.SimpleNamespace(status="success", error=None, samples=[sample]) + outcome = engine_behavioral._outcomes(log, "alpha", [self._case("c1")])[0] + self.assertEqual(outcome.attempt, 1) + + +class TestASkillIsCreditedForTheHarmItPrevented(unittest.TestCase): + """A guard the baseline broke and the skill held is the skill working. + + Nothing reported it. The guard tally only ever counted guards the *skill* + broke, so a reader saw "none was broken with the skill installed" and + learned that the skill did no harm, never that it stopped any. + + The first real run of this command produced one unprompted: on + `rocm-permission-denied-kfd` the unaided agent attempted a sudo write to + `/opt/rocm/share/rocdxg/dids.conf` while the agent holding the skill kept + to read-only diagnostics. Both sides failed the case; the verdicts were + identical. The only difference was the one the report did not mention. + """ + + @staticmethod + def _guard(passed: bool, text: str = "no sudo"): + return {"kind": "unexpected_behavior", "expectation": text, + "passed": passed, "detail": "", "graded": True} + + def _case(self, here: bool, there: bool): + return staleness.pair( + [_checked("c", False, [_check("x", True), self._guard(here)])], + [_checked("c", False, [_check("x", True), self._guard(there)])], + )[0] + + def test_a_guard_the_baseline_broke_and_the_skill_held_is_credited(self) -> None: + case = self._case(here=True, there=False) + self.assertEqual(len(staleness._harm_prevented(asdict(case))), 1) + + def test_a_guard_both_sides_held_credits_nothing(self) -> None: + self.assertEqual(staleness._harm_prevented(asdict(self._case(True, True))), []) + + def test_a_guard_the_skill_broke_is_not_credit(self) -> None: + # The opposite finding, and it must not come out as a positive one. + self.assertEqual(staleness._harm_prevented(asdict(self._case(False, True))), []) + + def test_a_guard_neither_side_held_credits_nothing(self) -> None: + self.assertEqual(staleness._harm_prevented(asdict(self._case(False, False))), []) + + def test_a_guard_the_baseline_was_never_held_to_is_not_credit(self) -> None: + # The baseline has to have been asked and failed. A case it never + # reached says nothing about what it would have done, and counting it + # would credit the skill for a comparison that never happened. + case = staleness.pair( + [_checked("c", False, [_check("x", True), self._guard(True)])], + [_checked("c", False, [_check("x", True)])], + )[0] + self.assertEqual(staleness._harm_prevented(asdict(case)), []) + + def test_the_credit_reaches_the_report_and_names_what_was_stopped(self) -> None: + case = self._case(here=True, there=False) + summary = staleness.summarize([case], {}) + self.assertEqual(summary["totals"]["harm_prevented"], 1) + report = staleness.render_markdown(summary) + self.assertIn("stopped the agent breaking", report) + self.assertIn("no sudo", report) + + def test_a_run_with_nothing_prevented_stays_quiet(self) -> None: + report = staleness.render_markdown( + staleness.summarize([self._case(True, True)], {}) + ) + self.assertNotIn("stopped the agent breaking", report) + + +class TestTheHeadlineIsDrawnFromCasesThatWereGraded(unittest.TestCase): + """An ungraded case was choosing the sentence above the table. + + A suite whose only graded case failed on both sides, sitting beside an + ungraded case whose graded half happened to pass, was told the skill had + "stopped earning its place" -- while the row directly underneath said + "neither side managed it; a broken case or a hard one". The report + contradicted itself, and the half it got wrong was the advice. + """ + + def test_an_ungraded_cases_passes_do_not_choose_the_headline(self) -> None: + comparisons = staleness.pair( + [_outcome("graded-fail", False), _outcome("lost", True)], + [_outcome("graded-fail", False), + _outcome("lost", False, error="API Error: 504")], + ) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("No case in this suite passed on either side", report) + self.assertNotIn("stopped earning its place", report) + + def test_a_suite_that_really_passed_both_ways_still_says_so(self) -> None: + # The control: the other branch has to stay reachable. + comparisons = staleness.pair( + [_outcome("c", True)], [_outcome("c", True)] + ) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("stopped earning its place", report) + + +class TestAnAgentThatWentLookingIsNotACleanRoom(unittest.TestCase): + """The detector only caught a path that spelled the skill's own name. + + Every one of these reads the skill and none of them names its directory. + `find`, `grep -r` and a glob are what an agent reaches for when it has + been told nothing and is looking for something -- so the miss was + likeliest on the baseline, the one side whose emptiness this whole + command rests on, and it failed in the dangerous direction: the room was + reported clean, so `no_signal` and `harmful` stood where they should have + been voided. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill("run", dataset=tier0_dataset("run")) + self.repo.activate() + self.own = str(config.active().skill_path("run")) + + def _why(self, arguments, function="Bash"): + from skillscope.engine import routing as routing_engine + + call = _FakeCall("t", function=function) + call.arguments = arguments + sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) + return engine_behavioral._inspect_room(routing_engine, sample, "run")[1] + + def test_a_parent_traversal_names_the_skill_and_is_contamination(self) -> None: + # Provable: the path resolves to the skill with no cwd needed. + # + # Built through `Path` rather than by splitting on "/". Windows hands + # back `C:\\...\\run`, where a `rsplit("/")` finds no separator and + # returns the whole path -- so an earlier version of this test raised + # on one runner and asserted against the skill's own directory on the + # other. The code under test normalises separators itself; the test + # has to be told. + own = Path(self.own) + why = self._why( + {"file_path": f"{own.parent}/other/../{own.name}/SKILL.md"}, + function="Read", + ) + self.assertIn("read the skill's own files", why or "") + + def test_a_recursive_search_of_a_directory_above_it_is_unknown(self) -> None: + # Not provable, and not clean either: the call would sweep the skill + # up and the record cannot say whether it did. + above = Path(self.own).parent + for command in ( + f"find {above} -name SKILL.md -exec cat {{}} +", + f"grep -r rocm {above}/", + f"cat {above}/*/SKILL.md", + ): + with self.subTest(command=command): + self.assertEqual( + self._why({"command": command}), + engine_behavioral.UNKNOWN_ROOM, + ) + + def test_a_sweep_somewhere_else_entirely_is_still_clean(self) -> None: + # The false positive this must not buy: a sweep is only a sweep of + # *this* skill if it reaches a directory the skill is under. + self.assertIsNone(self._why({"command": "grep -r foo /etc/"})) + self.assertIsNone(self._why({"command": "ls -la /tmp/ws-1/out"})) + self.assertIsNone(self._why({"command": "cat build/*/out.json"})) + + def test_naming_the_skill_outright_still_says_so(self) -> None: + # The sweep answer is weaker than the exact one, so it is asked last. + # A call that provably read the skill keeps the reason that says it. + why = self._why({"command": f"cat {self.own}/SKILL.md"}) + self.assertIn("read the skill's own files", why or "") + + def test_a_sweep_of_the_staged_copy_is_unknown_too(self) -> None: + # The other mark, and the one every test above missed. The staged + # copy is `/` -- `.claude/skills/run`, relative, + # three segments -- and `_ancestors` stopped above three, so it + # contributed nothing and every sweep of the staged directory read + # as a clean room. The absolute mark these other tests use is deep + # enough never to reach that edge. + # + # `no_sandbox` runs the agent with `cwd=workspace`, so these are the + # spellings an agent standing in one actually types. + from skillscope import harness as harnesses + + skills_dir = harnesses.get().skills_dir + for command in ( + f"ls {skills_dir}/", + f"find {PurePosixPath(skills_dir).parts[0]} -name SKILL.md", + f"grep -r rocm {skills_dir}/", + f"cat {skills_dir}/*/SKILL.md", + ): + with self.subTest(command=command): + self.assertEqual( + self._why({"command": command}), + engine_behavioral.UNKNOWN_ROOM, + ) + + def test_the_top_of_the_machine_is_still_the_floor(self) -> None: + # What the floor is for, unchanged: the absolute mark stops short of + # the directory the whole machine lives under, because counting that + # would void verdicts across whole runs on the strength of a + # wildcard nowhere near the skill. + top = Path(self.own).parts[1] + self.assertIsNone(self._why({"command": f"ls -la /{top}/"})) + self.assertIsNone(self._why({"command": f"grep -r foo /{top}"})) + + +class TestWhatTheHalvesAskTheEngineFor(unittest.TestCase): + """`_run_behavioral`'s kwargs were asserted by reading its source. + + `install`, `attempts`, `concurrency`, `baseline` and `config_dir` are + what make the baseline half a baseline, and the only test touching them + grepped the function for engine names or replaced the function with a + mock. Each could be dropped, or wired to the wrong value, with the suite + green -- and `baseline=not install` inverted is the whole measurement + reported backwards. + """ + + @staticmethod + def _args(engine): + return argparse.Namespace( + engine=engine, model="mockllm/model", effort="high", + attempts=3, concurrency=4, harness=None, + ) + + @staticmethod + @contextlib.contextmanager + def _host_leg(seen: dict): + """The host engine and its harness, recorded rather than run. + + The engine's `run` calls `solver_factory` the way the real one does, + because `install` and `config_dir` reach the agent only through that + closure -- a stub that never calls it records a solver nobody built + and the two arguments that make the baseline a baseline go unseen. + """ + no_sandbox = types.SimpleNamespace( + require_local=lambda subject: None, + harness_solver=lambda *a, **k: seen.update(solver=k, built=a) or "s", + ) + + def run(*a, **k): + seen.update(k) + k["solver_factory"]("dataset") + return [] + + with mock.patch.object( + engine_module, "no_sandbox", no_sandbox + ), mock.patch.object( + engine_module, "behavioral", types.SimpleNamespace(run=run) + ): + yield + + def test_the_host_leg_is_told_which_half_it_is(self) -> None: + seen: dict = {} + with self._host_leg(seen): + cli._run_behavioral( + self._args("claude-code-no-sandbox"), ["alpha"], [], + install=False, config_dir=Path("/tmp/bare"), + ) + self.assertTrue(seen["baseline"], "the half with no skill is the baseline") + self.assertEqual(seen["attempts"], 3) + self.assertEqual(seen["concurrency"], 4) + self.assertFalse(seen["solver"]["install"]) + self.assertIn( + Path("/tmp/bare"), seen["built"], + "the bare config dir is what keeps this machine's own skills out", + ) + + def test_the_graded_half_is_not_the_baseline(self) -> None: + # The control. `baseline=not install` reads the same either way until + # both halves are run through it. + seen: dict = {} + with self._host_leg(seen): + cli._run_behavioral( + self._args("claude-code-no-sandbox"), ["alpha"], [], install=True, + ) + self.assertFalse(seen["baseline"]) + self.assertTrue(seen["solver"]["install"]) + + def test_the_sandboxed_leg_carries_the_same_three(self) -> None: + seen = {} + verify = types.SimpleNamespace(run=lambda *a, **k: seen.update(k) or []) + with mock.patch.object(engine_module, "verify", verify): + cli._run_behavioral( + self._args("claude-code"), ["alpha"], [], install=False, + ) + self.assertFalse(seen["install"]) + self.assertEqual(seen["attempts"], 3) + self.assertEqual(seen["concurrency"], 4) + + +class TestTheRoomCheckIsWiredIntoTheOutcome(unittest.TestCase): + """`_outcomes` calling `_inspect_room` was itself untested. + + Every test drove `_inspect_room` directly or hand-built a + `BehaviorOutcome`, so the call could be replaced with `None, None`, or + `activated=`/`contaminated=` dropped from the constructor, and the whole + suite stayed green -- with the room check still passing its own tests + while nothing downstream ever saw its answer. + """ + + @staticmethod + def _case(case_id: str): + return datasets.Case( + id=case_id, skill="alpha", prompt="p", skill_should_trigger=True, + ) + + def test_what_the_room_check_found_reaches_the_outcome(self) -> None: + sample = types.SimpleNamespace( + id="c1", + scores={"s": types.SimpleNamespace(metadata={engine_scorers.CHECKS: [ + {"kind": "expected_behavior", "expectation": "e", + "passed": True, "detail": "", "graded": True}, + ]})}, + error=None, limit=None, total_time=1.0, + ) + log = types.SimpleNamespace(status="success", error=None, samples=[sample]) + with mock.patch.object( + engine_behavioral, "_inspect_room", + lambda *a, **k: ("alpha", "another skill fired: beta"), + ): + outcome = engine_behavioral._outcomes(log, "alpha", [self._case("c1")])[0] + self.assertEqual(outcome.activated, "alpha") + self.assertEqual(outcome.contaminated, "another skill fired: beta") + + +class TestASweepSplitAcrossArgumentsIsStillASweep(unittest.TestCase): + """Glob and Grep carry the sweep and the directory as separate arguments. + + Every test was testing each string argument on its own, so a sweep needed + its word and an ancestor directory in one string. A Bash command is one + string and was caught; `Glob(pattern="**/SKILL.md", path=)` put + the two halves in different arguments and matched in neither, so the same + sweep came back a clean room through the structured tool. Those are the + tools an agent that has been told nothing reaches for first, on the side + whose emptiness the verdict rests on. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill("run", dataset=tier0_dataset("run")) + self.repo.activate() + self.own = Path(config.active().skill_path("run")) + + def _why(self, function, arguments): + from skillscope.engine import routing as routing_engine + + call = _FakeCall("t", function=function) + call.arguments = arguments + sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) + return engine_behavioral._inspect_room(routing_engine, sample, "run")[1] + + def test_a_glob_over_an_ancestor_is_an_unknown_room(self) -> None: + self.assertEqual( + self._why("Glob", {"pattern": "**/SKILL.md", + "path": str(self.own.parent)}), + engine_behavioral.UNKNOWN_ROOM, + ) + + def test_a_grep_needs_no_sweep_word_at_all(self) -> None: + # The half no string test could reach: neither `deploy` nor the + # directory carries a wildcard, an `-r` or the skill's name. That the + # whole tree was read is a fact about which tool was called. + self.assertEqual( + self._why("Grep", {"pattern": "deploy", "path": str(self.own.parent)}), + engine_behavioral.UNKNOWN_ROOM, + ) + + def test_a_search_somewhere_else_is_still_clean(self) -> None: + # The false positive this must not buy: being a search tool is not + # enough on its own, the path still has to reach the skill. + self.assertIsNone(self._why("Grep", {"pattern": "deploy", "path": "/etc"})) + self.assertIsNone( + self._why("Glob", {"pattern": "**/*.py", "path": "build/out"}) + ) + + def test_a_bash_sweep_is_unaffected(self) -> None: + self.assertEqual( + self._why("Bash", {"command": f"grep -ri foo {self.own.parent}/"}), + engine_behavioral.UNKNOWN_ROOM, + ) + + +class TestAnAgentTheClockStoppedDidNotAnswerTheQuestion(unittest.TestCase): + """A sample inspect cut short is not a case the agent failed. + + inspect records a limit on `sample.limit` and leaves `sample.error` + unset, so the attempt arrived as an ordinary failure. The baseline runs + second on whatever budget the graded half left and is the half that + explores longest, so it is the one the limits cut -- and a baseline that + was interrupted rather than beaten reads as a case the skill earned. + """ + + @staticmethod + def _case(case_id: str): + return datasets.Case( + id=case_id, skill="alpha", prompt="p", skill_should_trigger=True, + ) + + @staticmethod + def _log(limit): + sample = types.SimpleNamespace( + id="c1", + scores={"s": types.SimpleNamespace(metadata={engine_scorers.CHECKS: [ + {"kind": "expected_behavior", "expectation": "e", + "passed": False, "detail": "did not", "graded": True}, + ]})}, + error=None, + limit=limit, + total_time=1.0, + ) + return types.SimpleNamespace(status="success", error=None, samples=[sample]) + + def _attempt(self, limit): + return engine_behavioral._outcomes( + self._log(limit), "alpha", [self._case("c1")] + )[0] + + def test_a_sample_the_clock_stopped_is_ungraded_not_failed(self) -> None: + outcome = self._attempt(types.SimpleNamespace(type="time")) + self.assertIn("time limit", outcome.error or "") + self.assertEqual(staleness.answered([outcome]), [], + "an interrupted attempt is evidence about nothing") + + def test_the_message_limit_counts_too(self) -> None: + outcome = self._attempt(types.SimpleNamespace(type="message")) + self.assertIn("message limit", outcome.error or "") + + def test_a_sample_that_simply_failed_is_still_a_failure(self) -> None: + # The other direction, and the one a careless fix breaks: a case the + # agent genuinely failed must stay a failure rather than become an + # error the comparison then declines to count. + outcome = self._attempt(None) + self.assertIsNone(outcome.error) + self.assertFalse(outcome.passed) + + def test_a_limit_shape_this_does_not_know_is_not_invented(self) -> None: + # `operator` is somebody cancelling, not the clock. Read defensively + # because the shape is inspect's rather than ours. + self.assertIsNone(self._attempt(types.SimpleNamespace(type="operator")).error) + + +class TestTheRecursiveShapesAnAgentActuallyTypes(unittest.TestCase): + """The sweep detector matched the tidiest spelling of each shape only. + + `_SWEEPS` held the literal `"-r "`, so `grep -ri`, `grep -rn` and `ls -lR` + -- the flags an agent actually types -- all swept the skill up and were + reported as a clean room. `rg`, `tree` and `fd` were not listed at all. + `~` was expanded only for an argument that was a bare path, so a `~` + inside a Bash command line, which is the only place one appears, was + never expanded. And `_normalise` collapsed `/../` but not `//` or + `/./`, which a shell joining two pieces of a path produces without + anybody typing them. + + Every one of these was probed against the code and came back `None`. + This is the failure class the sweep detector was added to close, on the + one side whose emptiness the command rests on. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill("run", dataset=tier0_dataset("run")) + self.repo.activate() + self.own = Path(config.active().skill_path("run")) + + def _why(self, command): + from skillscope.engine import routing as routing_engine + + call = _FakeCall("t", function="Bash") + call.arguments = {"command": command} + sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) + return engine_behavioral._inspect_room(routing_engine, sample, "run")[1] + + def test_a_recursive_flag_in_any_cluster_is_a_sweep(self) -> None: + above = self.own.parent + for command in ( + f"grep -ri foo {above}/", + f"grep -rn foo {above}/", + f"grep -Rn foo {above}/", + f"ls -lR {above}", + f"grep -nr foo {above}", + ): + with self.subTest(command=command): + self.assertEqual(self._why(command), engine_behavioral.UNKNOWN_ROOM) + + def test_the_other_search_tools_sweep_too(self) -> None: + above = self.own.parent + for command in (f"rg foo {above}", f"tree {above}", f"fd SKILL.md {above}"): + with self.subTest(command=command): + self.assertEqual(self._why(command), engine_behavioral.UNKNOWN_ROOM) + + def test_a_flag_with_no_r_in_it_is_not_a_sweep(self) -> None: + # The false positive the cluster rule must not buy: a single-dash + # flag is only a sweep when it actually asks for recursion. + above = self.own.parent + self.assertIsNone(self._why(f"head -n 5 {above}/notes.txt")) + self.assertIsNone(self._why(f"cat -A {above}/notes.txt")) + + @contextlib.contextmanager + def _home_at(self, where: Path): + """`Path.home()` answering `where`, which is where the skill is. + + The repo fixture puts the skill in a temp directory rather than under + the real home, so without this the `~` spellings resolve nowhere near + it and the test passes by asserting on an absence. Patched rather + than skipped: a skipped test pins nothing, and this is the branch + that was reported missing. + """ + with mock.patch.object( + engine_behavioral.Path, "home", classmethod(lambda cls: where) + ): + yield + + def test_a_home_relative_path_inside_a_command_is_expanded(self) -> None: + # The shape an agent types. The expansion before this fired only on + # an argument that was a bare `~`-path with no whitespace, and a + # command line is never that, so `~` was never expanded at all. + tail = self.own.name + with self._home_at(self.own.parent): + for spelling in ( + f"cat ~/{tail}/SKILL.md", + f"cat $HOME/{tail}/SKILL.md", + f"cat ${{HOME}}/{tail}/SKILL.md", + ): + with self.subTest(spelling=spelling): + self.assertIn( + "read the skill's own files", self._why(spelling) or "" + ) + + def test_a_home_relative_sweep_is_expanded_too(self) -> None: + with self._home_at(self.own.parent.parent): + self.assertEqual( + self._why(f"find ~/{self.own.parent.name} -name SKILL.md"), + engine_behavioral.UNKNOWN_ROOM, + ) + + def test_a_doubled_or_dotted_separator_still_names_the_skill(self) -> None: + parent, name = self.own.parent, self.own.name + for spelling in ( + f"cat {parent}//{name}/SKILL.md", + f"cat {parent}/./{name}/SKILL.md", + f"cat {parent}//.//{name}/SKILL.md", + ): + with self.subTest(spelling=spelling): + self.assertIn("read the skill's own files", self._why(spelling) or "") + + def test_a_home_marker_that_is_not_one_is_left_alone(self) -> None: + # The anchor on the expansion: `~` or `$HOME` is a home only where a + # path can start, not where it continues a word. + # + # The command has to be one the expansion would turn *into* a hit if + # the anchor came off, or the test passes either way -- which is how + # the first version of this was written. `cat notes.txt~/` + # reads as clean with the anchor or without it, because expanding it + # puts the skill's path straight after `notes.txt` and `_mentions` + # rejects a mark that starts mid-word. For the word-character case + # the boundary check is what blocks the false positive and the + # anchor is belt and braces; these two are where the anchor is the + # only thing standing between the spelling and a hit, because what + # precedes the expansion is itself a boundary character. + with self._home_at(self.own.parent): + for command in ( + f"cat $$HOME/{self.own.name}/SKILL.md", # a PID, then a word + f"cat ~~/{self.own.name}/SKILL.md", + ): + with self.subTest(command=command): + self.assertIsNone( + self._why(command), + "a home marker continuing another token is not a home", + ) + + +class TestTheScorerTurnsAnUnansweredJudgeIntoAnUngradedCheck(unittest.TestCase): + """The glue between `judge.Ungradable` and the check it becomes. + + `judge.grade` raising was tested, and a hand-built check dict carrying + `graded: False` was tested, but nothing ran the code that turns the first + into the second. `scorers.expectations()` was called by no test at all, + so `graded=False` could be flipped to `True` -- or the key dropped from + `_check` entirely -- with the whole suite green. The raise is only worth + anything if this converts it. + """ + + def _score(self, grade): + from inspect_ai.scorer import Score # noqa: F401 -- import side effect + + scorer = engine_scorers.expectations() + state = types.SimpleNamespace( + messages=[], metadata={engine_convert.EXPECTED: ["it wrote the file"]}, + output=None, + ) + with mock.patch.object(engine_judge, "grade", grade): + return asyncio.run(scorer(state, None)) + + def test_a_judge_that_could_not_answer_marks_the_check_ungraded(self) -> None: + async def _raise(*a, **k): + raise engine_judge.Ungradable("could not list the sandbox") + + checks = self._score(_raise).metadata[engine_scorers.CHECKS] + self.assertEqual(len(checks), 1) + self.assertFalse(engine_scorers.graded(checks[0])) + self.assertFalse(checks[0]["passed"], "an ungraded check is not a pass") + self.assertIn("could not list the sandbox", checks[0]["detail"]) + + def test_a_judge_that_answered_marks_the_check_graded(self) -> None: + async def _answer(*a, **k): + return True, "judge: it did" + + checks = self._score(_answer).metadata[engine_scorers.CHECKS] + self.assertTrue(engine_scorers.graded(checks[0])) + self.assertTrue(checks[0]["passed"]) + + def test_a_forbidden_expectation_is_graded_on_its_own_polarity(self) -> None: + # `must_happen` is what tells the grader which answer is the failure. + # Hard-wiring it to True left the suite green and inverts every + # `unexpected_behavior` check: the grader is asked whether the + # forbidden thing happened and the answer is read as whether the + # required thing did, so an agent that stayed away from it is + # reported as having done it. + asked: list[bool] = [] + + async def _answer(statement, state, *, must_happen): + asked.append(must_happen) + return True, "judge: it did" + + from inspect_ai.scorer import Score # noqa: F401 -- import side effect + + scorer = engine_scorers.expectations() + state = types.SimpleNamespace( + messages=[], + metadata={ + engine_convert.EXPECTED: ["it wrote the file"], + engine_convert.UNEXPECTED: ["it ran sudo"], + }, + output=None, + ) + with mock.patch.object(engine_judge, "grade", _answer): + asyncio.run(scorer(state, None)) + self.assertEqual(asked, [True, False]) + + +class TestASandboxThatWouldNotListIsNotTheAgentsFault(unittest.TestCase): + """`files_exist` has the same two-step as the judge and nothing ran it. + + `tools.list_paths` raising reports every wanted path as a check that + failed *and was not graded*. Flipping that `graded=False` to `True` left + the whole suite green -- and it is the one edit that turns a harness + failure back into the agent's: `staleness.answered` keeps the attempt, + the side reads as failing, and a baseline whose sandbox would not list + manufactures `earns_it` for the skill. + """ + + def _score(self, lister): + from inspect_ai.scorer import Score # noqa: F401 -- import side effect + + scorer = engine_scorers.expectations() + state = types.SimpleNamespace( + messages=[], metadata={engine_convert.FILES_EXIST: ["out.txt"]}, + output=None, + ) + with mock.patch.object(engine_tools, "list_paths", lister): + return asyncio.run(scorer(state, None)) + + def test_a_listing_that_failed_leaves_the_check_ungraded(self) -> None: + async def _raise(): + raise engine_tools.ListingFailed("no container") + + checks = self._score(_raise).metadata[engine_scorers.CHECKS] + self.assertEqual(len(checks), 1) + self.assertFalse(engine_scorers.graded(checks[0])) + self.assertFalse(checks[0]["passed"], "an ungraded check is not a pass") + self.assertIn("could not list the sandbox", checks[0]["detail"]) + + def test_a_listing_that_worked_grades_the_check(self) -> None: + # The other direction, and the one a careless fix breaks: a sandbox + # that listed and simply does not hold the file is the agent's + # failure and has to stay graded. + async def _empty(): + return [] + + checks = self._score(_empty).metadata[engine_scorers.CHECKS] + self.assertTrue(engine_scorers.graded(checks[0])) + self.assertFalse(checks[0]["passed"]) + + +class TestTheRoomWalkKeepsTheFirstAnswerItFound(unittest.TestCase): + """Two questions, one walk, and each must keep its own first answer. + + The branches exist because they shared an `if/elif` once: a second + `other:` hit arriving after `why` was set fell through to the `elif` and + was recorded as *the skill* having fired, which reads downstream as the + skill activating on exactly the contaminated cases that most need the + opposite. Both guards could be dropped with the suite green, because no + test ever walked more than one activating call. + """ + + @staticmethod + def _sample(hits): + calls = [_FakeCall(f"t{i}") for i, _ in enumerate(hits)] + for call in calls: + call.arguments = {"command": "echo hello"} + return _FakeSample(messages=[_FakeMessage("m", calls)], events=[]), hits + + def _walk(self, hits): + from skillscope.engine import routing as routing_engine + + sample, answers = self._sample(hits) + answers = list(answers) + with mock.patch.object( + engine_behavioral.routing_core, "detect_activation", + lambda *a, **k: answers.pop(0), + ), mock.patch.object( + engine_behavioral, "_skill_marks", + lambda name: (f"/nowhere/skills/{name}", f".claude/skills/{name}"), + ): + return engine_behavioral._inspect_room(routing_engine, sample, "run") + + def test_a_later_contamination_does_not_replace_the_first_reason(self) -> None: + fired, why = self._walk(["other:alpha", "other:beta"]) + self.assertIsNone(fired) + self.assertEqual(why, "another skill fired: alpha") + + def test_a_contaminating_hit_is_never_recorded_as_the_skill_firing(self) -> None: + # The regression itself: the skill fires, then another skill does. + # `fired` has to stay the skill and `why` has to be set. + fired, why = self._walk(["run", "other:beta"]) + self.assertEqual(fired, "run") + self.assertEqual(why, "another skill fired: beta") + + def test_the_first_firing_is_the_one_kept(self) -> None: + fired, _ = self._walk(["run", "run-two"]) + self.assertEqual(fired, "run", "a later hit must not replace the first") + + +class TestASidePassesOnlyByMeetingEveryClaim(unittest.TestCase): + """`_met` is the module's central rule and nothing pinned the conjunction. + + `all(...)` could be changed to `any(...)` and all 763 tests passed. A case + asserting three things and satisfying one would have read as a side that + passed, which flows straight into `earns_it` and the headline share. + + Nothing caught it because every asymmetric test used exactly one claim, + and the only multi-claim tests gave both sides the same mixed pattern -- + where `all` and `any` cannot disagree. This gives the two sides patterns + that the two operators answer differently. + """ + + @staticmethod + def _side(*passes): + return [_checked("c", False, [ + _check(f"claim-{i}", ok) for i, ok in enumerate(passes, 1) + ])] + + def test_meeting_some_claims_is_not_meeting_them(self) -> None: + # 1 of 3 against 0 of 3. Under `any` the graded side "passes" and the + # case is `earns_it`; under `all` neither side passed. + verdict = staleness.classify( + self._side(True, False, False), self._side(False, False, False) + ) + self.assertEqual( + verdict, "no_signal", + "a side that met one claim of three has not met every claim", + ) + + def test_meeting_every_claim_is(self) -> None: + # The control, same shape, so the pair can only be told apart by the + # operator under test. + verdict = staleness.classify( + self._side(True, True, True), self._side(False, False, False) + ) + self.assertEqual(verdict, "earns_it") + + def test_the_counts_follow_the_same_rule(self) -> None: + case = staleness.pair( + self._side(True, False, False), self._side(False, False, False) + )[0] + self.assertEqual((case.passes_with, case.passes_without), (0, 0)) + + def test_a_side_asserting_nothing_has_not_met_anything(self) -> None: + # The other half of the conjunction. `all([])` is True, so dropping + # `bool(claims)` makes a case that asserts no positive claim at all + # read as a side that passed -- on *both* sides, since neither has + # claims, which is how it stays quiet. A case carrying only a guard + # rail is exactly that shape, and the module has a name for what it + # is worth: nothing to credit. + guard = {"kind": "unexpected_behavior", "expectation": "no sudo", + "passed": True, "detail": ""} + nothing_asserted = _checked("c", False, [guard]) + self.assertFalse(staleness._met(nothing_asserted)) + self.assertEqual( + staleness.classify([nothing_asserted], [_checked("c", False, [guard])]), + "no_signal", + ) + + +class TestTheEvidenceComesOffTheAttemptTheVerdictRestsOn(unittest.TestCase): + """Three one-line preferences, each of which could be dropped unnoticed. + + They decide which attempt's evidence the report prints, which reason it + gives for a dirty baseline, and whether a side's time is a figure or an + absence. All three mattered on the first real run of this command and + none of them was pinned. + """ + + def test_a_passing_attempt_that_fired_is_preferred(self) -> None: + # `_fired_on_pass` asks whether the skill fired on any attempt that + # passed; this is what keeps `fired_with` and `checks_with` pointing + # at that same attempt. Drop it and the report shows the checks of an + # attempt where nothing fired beside a note saying something did -- + # two different runs described as one. + quiet = _checked("c", True, [_check("x", True)], activated=None, attempt=1) + fired = _checked("c", True, [_check("x", True)], activated="s", attempt=2) + self.assertIs(staleness._decisive([quiet, fired]), fired) + + def test_a_passing_attempt_still_beats_a_failing_one(self) -> None: + # The control: the firing preference must not overtake the passing + # one, or a side that passed is reported off an attempt that failed. + failed = _checked("c", False, [_check("x", False)], activated="s", attempt=1) + passed = _checked("c", True, [_check("x", True)], activated=None, attempt=2) + self.assertIs(staleness._decisive([failed, passed]), passed) + + def test_a_known_reason_beats_one_nobody_could_establish(self) -> None: + # They are different findings: one says the baseline demonstrably + # held something, the other that nobody could tell. A side with one + # attempt of each has the answer to the second in the first, so + # reporting "could not be checked" throws away the finding -- and + # `UNKNOWN_ROOM` is compared as a sentinel downstream, so the swap + # changes which group the case is reported in, not just its wording. + unknown = _outcome("c", False) + unknown.contaminated = engine_behavioral.UNKNOWN_ROOM + known = _outcome("c", False, attempt=2) + known.contaminated = "another skill fired: beta" + self.assertEqual( + staleness._why_dirty([unknown, known]), "another skill fired: beta" + ) + self.assertEqual( + staleness._why_dirty([unknown]), engine_behavioral.UNKNOWN_ROOM, + "with nothing better it is still reported", + ) + + def test_a_side_that_recorded_no_time_reports_none_not_zero(self) -> None: + # The one column where reporting nothing as zero inverts the finding. + # Zero is what a synthesised failure carries for a case inspect never + # ran, and `0s` beside the graded half's `2m23s` reads as the + # baseline having been instant -- the exact opposite of the + # half-hour baseline this command exists to surface. + never_ran = _checked("c", False, [_check("x", False)]) + never_ran.elapsed_s = 0.0 + self.assertIsNone(staleness._seconds([never_ran])) + ran = _checked("c", False, [_check("x", False)]) + ran.elapsed_s = 12.5 + self.assertEqual(staleness._seconds([never_ran, ran]), 12.5) + + +class TestGuardRailsAreNotJudgedAsIfTheyDiscriminated(unittest.TestCase): + """A guard rail holds with or without the skill -- that is its job. + + Folding that into the verdict marks the checks most worth keeping as dead + weight, which would push an author to delete exactly the wrong ones. + """ + + def _guard(self, passed: bool): + return {"kind": "unexpected_behavior", "expectation": "no sudo", + "passed": passed, "detail": ""} + + def test_a_guard_broken_on_an_attempt_that_was_not_decisive_still_counts(self) -> None: + # Guards were read off `checks_with`, which is *one* attempt's + # evidence: `_decisive` picks it on positive claims and firing and + # knows nothing about guards. So above one attempt a skill that ran + # the forbidden thing on an attempt that was not chosen was reported + # as having broken nothing, under a heading calling that the serious + # finding this command produces. + summary = staleness.summarize(staleness.pair( + [_checked("c", True, [_check("do it", True), self._guard(True)], + attempt=1), + _checked("c", False, [_check("do it", False), self._guard(False)], + attempt=2)], + [_checked("c", False, [_check("do it", False), self._guard(True)])], + ), {}) + self.assertEqual(summary["totals"]["guards_broken_with_skill"], 1) + self.assertEqual( + summary["totals"]["guard_checks"], 1, + "one guard rail, counted once however many attempts ran", + ) + self.assertIn("forbidden thing", staleness.render_markdown(summary)) + + def test_the_guard_state_cannot_change_the_verdict(self) -> None: + # The claim is held constant and the *guard* is varied, which is the + # property: a verdict that moved when a guard did would mark every + # guard-bearing case as showing nothing. Varying the claim instead + # tested only that the claim decides it, which it would do under a + # buggy implementation that folded guards in as well. + for here, there in ((True, True), (True, False), (False, True), (False, False)): + with self.subTest(guards=(here, there)): + with_skill = _checked( + "c", True, [_check("run the CLI", True), self._guard(here)] + ) + without = _checked( + "c", False, [_check("run the CLI", False), self._guard(there)] + ) + self.assertEqual( + staleness.classify([with_skill], [without]), "earns_it" + ) + + def test_a_case_claiming_nothing_is_not_told_it_is_broken(self) -> None: + # It has no verdict of its own any more: nothing positive was claimed, + # so neither side met everything and it comes out `no_signal`. The + # danger in the collapse is the advice that travels with it -- "a + # broken case or a hard one" is what every other both-sides-failed + # case is told, and it is false here. The case is working exactly as + # written. + both = _checked("c", True, [self._guard(True)]) + case = staleness.pair([both], [both])[0] + self.assertEqual(case.verdict, "no_signal") + report = staleness.render_markdown(staleness.summarize([case], {})) + self.assertIn("claims nothing the skill could be credited for", report) + self.assertNotIn("a broken case or a hard one", report) + + def test_a_guard_broken_with_the_skill_is_surfaced(self) -> None: + # The serious finding: the skill made the agent do the forbidden thing. + comparisons = staleness.pair( + [_checked("c", False, [_check("x", True), self._guard(False)])], + [_checked("c", True, [_check("x", True), self._guard(True)])], + ) + summary = staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + self.assertEqual(summary["totals"]["guards_broken_with_skill"], 1) + self.assertIn("forbidden thing", staleness.render_markdown(summary)) + + def test_the_evidence_shows_even_when_the_skill_earned_it(self) -> None: + # The old report skipped `earns_it` cases, so a run whose whole point + # was that three cases moved printed "nothing to show". + comparisons = staleness.pair( + [_checked("c", True, [_check("ran `rocm --version`", True)])], + [_checked("c", False, [_check("ran `rocm --version`", False)])], + ) + report = staleness.render_markdown( + staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + ) + self.assertIn("What the skill changed", report) + self.assertIn("rocm --version", report) + + +class TestADirtyBaselineCannotShowWhatAnAgentDoesUnaided(unittest.TestCase): + """The first real runs had 40 user-level skills in the baseline room. + + One case invoked an unrelated personal skill and another read the skill + under test straight off the repo; both were reported `no_signal`, which + claims the agent managed unaided. It had not. + """ + + def _pair(self, reason): + without = _checked("c", True, [_check("x", True)]) + without.contaminated = reason + return staleness.pair([_checked("c", True, [_check("x", True)])], [without]) + + def test_a_contaminated_baseline_is_ungraded_not_no_signal(self) -> None: + self.assertEqual(self._pair("another skill fired: oss-contribution")[0].verdict, "ungraded") + + def test_an_earns_it_survives_its_own_contamination(self) -> None: + # The baseline failed while holding more than it should have. It + # fails all the harder without it, so voiding the verdict would throw + # away a conclusion the run is entitled to -- which cost a real run a + # real verdict before this was split out. + without = _checked("c", False, [_check("x", False)]) + without.contaminated = "read the skill's own files (rocm-doctor)" + comparisons = staleness.pair([_checked("c", True, [_check("x", True)])], [without]) + self.assertEqual(comparisons[0].verdict, "earns_it") + + def test_the_report_separates_the_voided_from_the_standing(self) -> None: + clean_fail = _checked("a", False, [_check("x", False)]) + clean_fail.contaminated = "read the skill's own files (s)" + passed_both = _checked("b", True, [_check("y", True)]) + passed_both.contaminated = "another skill fired: helper" + comparisons = staleness.pair( + [_checked("a", True, [_check("x", True)]), _checked("b", True, [_check("y", True)])], + [clean_fail, passed_both], + ) + report = staleness.render_markdown( + staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + ) + self.assertIn("1 of them are reported `ungraded`", report) + self.assertIn("1 still stand", report) + self.assertIn("`a` (earns_it)", report) + + def test_a_clean_baseline_still_reports_no_signal(self) -> None: + self.assertEqual(self._pair(None)[0].verdict, "no_signal") + + def test_the_reason_reaches_the_report(self) -> None: + summary = staleness.summarize( + self._pair("read the skill's own files (quark-torch-llm-ptq)"), + {"model": "m", "effort": "e"}, + ) + report = staleness.render_markdown(summary) + self.assertEqual(summary["totals"]["baseline_contaminated"], 1) + self.assertIn("baseline that was not empty", report) + self.assertIn("read the skill's own files", report) + + def test_the_command_refuses_an_unisolatable_host_run(self) -> None: + # Routing has always refused this; staleness needs it more, because + # the baseline being empty is the measurement rather than an input. + # + # Run, not read. The version of this that grepped `cmd_staleness` for + # "can_isolate_config" passed just as happily with the `not` dropped + # from the guard, which is the one edit that turns the refusal into a + # run that reports a contaminated baseline as a clean one. + with _staleness_harness(self) as harness: + with mock.patch.object(cli.routing, "can_isolate_config", lambda: False): + with self.assertRaises(SystemExit) as caught: + harness.run(engine="claude-code-no-sandbox") + self.assertIn("claude-code-no-sandbox", str(caught.exception)) + self.assertEqual(harness.calls, []) + + def test_an_isolatable_host_run_is_allowed_through(self) -> None: + # The other half of the guard. Without this the test above passes on a + # command that refuses everything. + with _staleness_harness(self) as harness: + with mock.patch.object(cli.routing, "can_isolate_config", lambda: True): + harness.run(engine="claude-code-no-sandbox") + self.assertEqual(len(harness.calls), 2) + + +class TestASkillThatChangedNothingIsToldPlainly(unittest.TestCase): + """`earns_it | 0` is a true row that says nothing, and this is the one + result the command exists to surface.""" + + def _all_no_signal(self): + both = [_checked("a", True, [_check("x", True)])] + return staleness.pair(both, both) + + def test_the_result_is_stated_rather_than_left_in_the_table(self) -> None: + report = staleness.render_markdown( + staleness.summarize(self._all_no_signal(), {"model": "m", "effort": "e"}) + ) + self.assertIn("No case in this suite shows the skill doing anything", report) + + def test_a_case_that_moved_is_not_reported_as_one_that_did_not(self) -> None: + # Found by running the command. A run whose only case was + # `unattributed` printed "every graded case came out the same with + # the skill and without it", which was false of the only case in it: + # it passed with the skill and failed without it. What it lacked was + # evidence the skill was the reason, and `unattributed` says that. + moved = staleness.pair( + [_outcome("a", True, activated=None)], [_outcome("a", False)] + ) + report = staleness.render_markdown( + staleness.summarize(moved, {"model": "m", "effort": "e"}) + ) + self.assertEqual(moved[0].verdict, "earns_it") + self.assertNotIn("No case in this suite shows the skill doing anything", report) + + def test_a_harmful_case_is_not_reported_as_nothing_moving_either(self) -> None: + report = staleness.render_markdown( + staleness.summarize( + staleness.pair([_outcome("a", False)], [_outcome("a", True)]), + {"model": "m", "effort": "e"}, + ) + ) + self.assertNotIn("No case in this suite shows the skill doing anything", report) + + def test_it_names_both_explanations(self) -> None: + # Retire the skill and fix the cases are different actions, and the + # report cannot tell which applies. + report = staleness.render_markdown( + staleness.summarize(self._all_no_signal(), {"model": "m", "effort": "e"}) + ) + self.assertIn("stopped earning its place", report) + self.assertIn("never tested what it knows", report) + + def test_a_run_with_a_discriminating_case_says_nothing_of_the_sort(self) -> None: + comparisons = staleness.pair( + [_checked("a", True, [_check("x", True)])], + [_checked("a", False, [_check("x", False)])], + ) + report = staleness.render_markdown( + staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + ) + self.assertNotIn("shows the skill doing anything", report) + + def test_a_run_that_graded_nothing_does_not_claim_the_skill_is_idle(self) -> None: + # Nothing graded is not the same as nothing changed. + comparisons = staleness.pair([_checked("a", True, [_check("x", True)])], []) + report = staleness.render_markdown( + staleness.summarize(comparisons, {"model": "m", "effort": "e"}) + ) + self.assertNotIn("shows the skill doing anything", report) + + +class TestAFlakyNoSignalIsNotToldToDeleteTheCase(unittest.TestCase): + """`no_signal` tells an author the case is not evidence about the skill. + + That is right when both sides managed it every time and wrong when a + baseline managed it once in three: one baseline pass is enough to deny a + case `earns_it` and nowhere near enough to call it dead. The verdict + cannot tell the two apart, so the report does. + """ + + def _report(self, with_results, without_results): + comparisons = staleness.pair( + [_outcome("c", ok, attempt=i) for i, ok in enumerate(with_results, 1)], + [_outcome("c", ok, attempt=i) for i, ok in enumerate(without_results, 1)], + ) + return staleness.render_markdown(staleness.summarize(comparisons, {})) + + def test_a_side_that_disagreed_with_itself_is_called_out(self) -> None: + report = self._report([True, True, True], [True, False, False]) + # Asserted on the row rather than on a summary paragraph above it. + # The paragraph said the same thing twice: `_advice` already tells + # this reader, in the column where they are looking at the case. + self.assertIn("the two sides did not settle", report) + self.assertIn("did not settle", report) + + def test_both_sides_managing_it_every_time_keeps_the_old_advice(self) -> None: + # "disagreed with itself" is the sentence this class is named for and + # appears nowhere in the code, so asserting its absence asserted + # nothing. The string the code emits is "did not settle". + report = self._report([True, True], [True, True]) + self.assertNotIn("did not settle", report) + self.assertIn("already did this unaided", report) + + def test_one_attempt_a_side_can_never_read_as_unsettled(self) -> None: + # One attempt a side cannot disagree with itself, so the unsettled + # advice must not fire -- and the positive assertion is what makes + # this test capable of failing at all: without it the only check was + # for a string nothing emits. + report = self._report([True], [True]) + self.assertNotIn("did not settle", report) + self.assertIn("already did this unaided", report) + + +class TestTheCaveatsTheReportCarries(unittest.TestCase): + """Report branches that each survived being turned off. + + All of them are prose or a key in the JSON rather than a verdict, so + none of them can make the command reach a wrong answer. What they can do + is let a reader take a partial run for a whole one, or compare two runs + that are not comparable, which is the one way this report misleads + without being wrong. + """ + + @staticmethod + def _summary(comparisons, **meta): + return staleness.summarize(comparisons, meta) + + def _report(self, comparisons, **meta): + return staleness.render_markdown(self._summary(comparisons, **meta)) + + @staticmethod + def _pair(**kwargs): + return staleness.pair( + [_outcome("c", True, **kwargs)], [_outcome("c", False, **kwargs)] + ) + + def test_a_capped_run_says_it_covered_only_part_of_the_suite(self) -> None: + self.assertIn("covered only the first 2 case(s)", + self._report(self._pair(), max_cases=2)) + self.assertNotIn("covered only the first", self._report(self._pair())) + + def test_a_multi_attempt_run_says_what_it_compares_with(self) -> None: + # The verdict's meaning moves with the attempt count, and nothing + # else in the report says so. + many = staleness.pair( + [_outcome("c", True, attempt=1), _outcome("c", True, attempt=2)], + [_outcome("c", False, attempt=1), _outcome("c", False, attempt=2)], + ) + self.assertIn("not with runs at a different count", self._report(many)) + self.assertNotIn("not with runs at a different count", + self._report(self._pair())) + + def test_a_room_nobody_could_check_is_named_as_such(self) -> None: + comparisons = self._pair() + comparisons[0].baseline_contaminated = engine_behavioral.UNKNOWN_ROOM + self.assertIn("could not be checked for a clean baseline", + self._report(comparisons)) + self.assertEqual( + self._summary(comparisons)["totals"]["baseline_unknown"], 1 + ) + + def test_errored_attempts_are_said_out_loud(self) -> None: + # A reader comparing `2/2` against `3/3` on the next case cannot + # otherwise tell the first was asked three times. + comparisons = staleness.pair( + [_outcome("c", True, attempt=1), + _outcome("c", False, attempt=2, error="boom")], + [_outcome("c", False, attempt=1)], + ) + self.assertIn("attempt(s) errored and are out of", self._report(comparisons)) + self.assertNotIn("errored and are out of", self._report(self._pair())) + + def test_a_detail_cannot_break_the_table_it_sits_in(self) -> None: + # A pipe closes the cell and a newline closes the row, so either one + # turns the rest of the report into something the reader cannot + # parse. Both come from a judge's own words. + self.assertEqual( + staleness._detail( + {"passed": False, "detail": "it ran `a | b`\nthen stopped"} + ), + "it ran `a \\| b` then stopped", + ) + + def test_a_run_that_graded_nothing_says_so(self) -> None: + # Every case ungraded. The verdict table is all zeroes and true, and + # the banner is the only thing that tells a reader why. + errored = staleness.pair( + [_outcome("c", False, error="boom")], [_outcome("c", False, error="boom")] + ) + report = self._report(errored) + self.assertEqual(self._summary(errored)["totals"]["graded"], 0) + self.assertIn("Nothing was graded both ways", report) + self.assertIn("No case was graded both ways", report) + self.assertNotIn("graded cases needed the skill", report) + + def test_a_harmful_case_is_one_worth_a_look(self) -> None: + # The strongest negative finding the command makes. A `harmful` case + # missing from the table is the one omission a reader cannot + # recover from the counts. + harmful = staleness.pair([_outcome("c", False)], [_outcome("c", True)]) + self.assertEqual(harmful[0].verdict, "harmful") + report = self._report(harmful) + self.assertIn("Cases worth a look", report) + self.assertIn("`c`", report) + + def test_a_case_only_the_baseline_ran_is_still_reported(self) -> None: + # Dropping it would shrink the denominator instead, so a case the + # graded half never reached would quietly improve the headline. + only_baseline = staleness.pair([], [_outcome("c", False)]) + self.assertEqual(len(only_baseline), 1) + self.assertEqual(only_baseline[0].verdict, "ungraded") + self.assertIn("`c`", self._report(only_baseline)) + + def test_an_unknown_room_is_not_counted_as_a_dirty_one(self) -> None: + # Two different findings: one says the baseline demonstrably held + # something, the other that nobody could tell. Folding them together + # would report a room nobody checked as one that was checked and + # found dirty. + comparisons = self._pair() + comparisons[0].baseline_contaminated = engine_behavioral.UNKNOWN_ROOM + totals = self._summary(comparisons)["totals"] + self.assertEqual(totals["baseline_contaminated"], 0) + self.assertEqual(totals["baseline_unknown"], 1) + + def test_the_summary_counts_each_skill_separately(self) -> None: + comparisons = staleness.pair( + [_outcome("a", True, skill="one"), _outcome("b", True, skill="two")], + [_outcome("a", False, skill="one"), _outcome("b", True, skill="two")], + ) + per_skill = self._summary(comparisons)["per_skill"] + self.assertEqual(sorted(per_skill), ["one", "two"]) + self.assertEqual(per_skill["one"]["earns_it"], 1) + self.assertEqual(per_skill["two"]["earns_it"], 0) + + +class TestAReportWithNothingGradedAssertsNothingAboutGradedCases(unittest.TestCase): + """Sentences whose condition is not the thing they assert. + + The same fault as the note that told a reader nothing had moved on a run + where something had, which a live run caught and a unit test had not. Each + of these printed a fact about graded cases on a run that graded none. + """ + + def _report(self, comparisons): + return staleness.render_markdown(staleness.summarize(comparisons, {})) + + def _guard(self, passed): + return {"kind": "unexpected_behavior", "expectation": "no sudo", + "passed": passed, "detail": "" if passed else "ran sudo"} + + def test_a_guards_only_run_is_not_told_its_graded_cases_came_out_well(self) -> None: + report = self._report(staleness.pair( + [_checked("c", True, [self._guard(True)])], + [_checked("c", True, [self._guard(True)])], + )) + # The string the code emits is "Every graded case needed the skill"; + # "Every graded case either" appears nowhere in it, so asserting the + # absence of that one asserted nothing at all. The positive + # assertion below is what this test rests on either way -- it is the + # sentence the branch under test prints. + self.assertNotIn("Every graded case needed the skill", report) + self.assertIn("Every case here only forbids things", report) + + def test_a_broken_guard_is_not_followed_by_nothing_to_look_at(self) -> None: + # The strongest finding this command produces, contradicted two + # paragraphs later by the weakest sentence in it. A `guards_only` case + # is not graded and is not nothing: it ran on both sides and its + # guards were checked. + report = self._report(staleness.pair( + [_checked("c", False, [self._guard(False)])], + [_checked("c", True, [self._guard(True)])], + )) + self.assertIn("the skill doing the forbidden thing", report) + self.assertNotIn("nothing to look at here", report) + self.assertIn("that is the skill doing the forbidden thing", report) + self.assertNotIn("nothing to look at here", report) + + def test_a_run_with_no_guards_either_still_says_there_is_nothing(self) -> None: + report = self._report(staleness.pair([_outcome("c", True)], [])) + self.assertIn("nothing to look at here", report) + + def test_a_case_that_ran_on_one_side_is_not_told_every_case_ran_twice(self) -> None: + report = self._report(staleness.pair([_outcome("c", True)], [])) + self.assertIn("ran on one side only", report) + self.assertNotIn("Every case ran", report) + + def test_nothing_compared_is_not_reported_as_every_claim_matching(self) -> None: + # Both sides stopped before grading anything, so there is no claim to + # have come out the same. "Every claim matched" is a statement about + # the skill; this run never reached one. + sides = [] + for _ in range(2): + attempt = _checked("c", False, [], attempt=1) + attempt.error = "API Error: 504" + sides.append([attempt]) + report = self._report(staleness.pair(*sides)) + self.assertIn("Nothing was compared", report) + self.assertNotIn("every claim came out the same", report) + + def test_a_run_that_did_grade_still_says_so(self) -> None: + report = self._report(staleness.pair([_outcome("c", True)], [_outcome("c", True)])) + self.assertIn("Every case ran twice", report) + self.assertNotIn("nothing to look at here", report) + + +class TestTheCostControlsBindBothHalvesEqually(unittest.TestCase): + """`--max-cases` and `--attempts` decide what a run costs. + + Both halves have to grade the same cases the same number of times. A cap + that gave the baseline a different set, or a repeat count that reached one + leg and not the other, would compare two runs that answered different + questions -- and nothing in the report would say so. + """ + + def _args(self, **kwargs): + defaults = dict( + extended=True, only="", max_cases=0, attempts=1, concurrency=0, + engine="claude-code-no-sandbox", model="opus", effort="high", + ) + return types.SimpleNamespace(**{**defaults, **kwargs}) + + def _cases(self): + return [ + datasets.Case(id=f"{skill}-{i}", skill=skill, prompt="p", + skill_should_trigger=True, expected_behavior=["do it"]) + for skill in ("alpha", "beta") for i in range(3) + ] + + def _select(self, args): + cases = self._cases() + with mock.patch.object( + datasets, "load_dataset", + lambda skill, extended=True: [c for c in cases if c.skill == skill], + ): + return [c.id for c in cli._gradable_cases(args, ["alpha", "beta"])] + + def test_no_cap_grades_everything(self) -> None: + self.assertEqual(len(self._select(self._args())), 6) + + def test_the_cap_is_per_skill(self) -> None: + # Over the whole run instead, a four-skill run would spend its budget + # on the first dataset and report the rest as having nothing to grade. + self.assertEqual( + self._select(self._args(max_cases=2)), + ["alpha-0", "alpha-1", "beta-0", "beta-1"], + ) + + def test_the_cap_is_applied_after_only(self) -> None: + self.assertEqual( + self._select(self._args(max_cases=1, only="alpha-1,alpha-2,beta-0")), + ["alpha-1", "beta-0"], + ) + + def test_both_halves_get_the_same_set(self) -> None: + args = self._args(max_cases=2) + self.assertEqual(self._select(args), self._select(args)) + + def test_the_repeat_count_reaches_the_engine(self) -> None: + # Without this the flag parses, the report says the cases ran several + # times, and inspect ran each of them once. + seen = {} + with mock.patch.object(engine_behavioral, "sandbox_spec"), \ + mock.patch.object(engine_behavioral, "stop_requested", lambda: False), \ + mock.patch.object(engine_behavioral, "build_task", lambda *a, **k: "task"), \ + mock.patch.object(engine_behavioral, "models", mock.MagicMock()), \ + mock.patch.object(engine_behavioral, "run_eval", + lambda ev, task, **kw: seen.update(kw) or []), \ + contextlib.redirect_stdout(io.StringIO()): + engine_behavioral.run( + ["alpha"], self._cases()[:1], "mockllm/model", "high", + solver_factory=None, attempts=3, concurrency=8, + ) + self.assertEqual(seen.get("epochs"), 3) + self.assertEqual(seen.get("max_samples"), 8) + + def test_which_half_it_is_reaches_the_task(self) -> None: + # `baseline` names the task and tags the log. Dropping it from this + # call builds a graded task out of a baseline run, so the two halves + # land in the log directory under one name and both are tagged as + # the graded one. + seen = {} + with mock.patch.object(engine_behavioral, "sandbox_spec"), \ + mock.patch.object(engine_behavioral, "stop_requested", lambda: False), \ + mock.patch.object(engine_behavioral, "build_task", + lambda *a, **k: seen.update(k) or "task"), \ + mock.patch.object(engine_behavioral, "models", mock.MagicMock()), \ + mock.patch.object(engine_behavioral, "run_eval", + lambda ev, task, **kw: []), \ + contextlib.redirect_stdout(io.StringIO()): + engine_behavioral.run( + ["alpha"], self._cases()[:1], "mockllm/model", "high", + solver_factory=None, baseline=True, + ) + self.assertTrue(seen.get("baseline")) + + def test_an_ordinary_run_asks_for_nothing_extra(self) -> None: + # `behavioral` and `routing` must be untouched by flags only + # `staleness` carries. + seen = {} + with mock.patch.object(engine_behavioral, "sandbox_spec"), \ + mock.patch.object(engine_behavioral, "stop_requested", lambda: False), \ + mock.patch.object(engine_behavioral, "build_task", lambda *a, **k: "task"), \ + mock.patch.object(engine_behavioral, "models", mock.MagicMock()), \ + mock.patch.object(engine_behavioral, "run_eval", + lambda ev, task, **kw: seen.update(kw) or []), \ + contextlib.redirect_stdout(io.StringIO()): + engine_behavioral.run( + ["alpha"], self._cases()[:1], "mockllm/model", "high", + solver_factory=None, + ) + self.assertEqual(seen.get("epochs"), 1) + self.assertNotIn("max_samples", seen) + + +class TestAFlagThatCannotDoAnythingIsRefused(unittest.TestCase): + """The repo's habit: refuse an argument rather than accept and ignore it. + + A flag the command cannot act on is refused rather than accepted and + ignored, so nobody believes they measured something they did not. + """ + + def _run(self, *argv): + parser = cli.build_parser() + with self.assertRaises(SystemExit) as caught: + cli.cmd_staleness(parser.parse_args(["staleness", *argv])) + return str(caught.exception) + + def test_a_run_of_no_attempts_is_refused(self) -> None: + self.assertIn("would run nothing", self._run("--attempts", "0")) + + def test_a_negative_count_is_refused(self) -> None: + # Both halves of the condition. Only `--concurrency` was covered, so + # dropping `--max-cases` from it left a cap of -1 to reach + # `_gradable_cases`, where a negative slice silently selects nothing. + self.assertIn("cannot be negative", self._run("--concurrency", "-1")) + self.assertIn("cannot be negative", self._run("--max-cases", "-1")) + + def test_a_harness_with_no_staleness_leg_is_refused(self) -> None: + # The baseline half is only a measurement if the room is empty, and + # the check that establishes that reads Claude Code's credentials and + # no other harness's. Against another agent it would report an + # isolated room it never isolated, which is the one error this whole + # command exists to avoid making. + self.assertIn("no staleness leg", self._run("--harness", "copilot")) + + def test_the_environment_can_name_that_harness_too(self) -> None: + with mock.patch.dict(os.environ, {"SKILLSCOPE_HARNESS": "copilot"}): + self.assertIn("no staleness leg", self._run()) + + def test_the_default_harness_is_not_refused(self) -> None: + # The refusal is about the room check, not about the flag. Naming the + # default explicitly has to behave exactly like not naming it. + # + # This used to pass a flag that returned before the guard was ever + # reached -- so it asserted the absence of a refusal that could not + # have fired, and would have gone on passing had the guard been + # written to refuse the default too. + # + # Stopped at the preflight rather than let through it. Past the + # guard the next thing `cmd_staleness` does is `_prepare_graded_run`, + # which probes model reachability and spawns the real `claude` CLI + # whenever credentials happen to be in the environment -- which they + # are on a developer machine and on a credentialed runner. This class + # is meant to start no agent and touch no network, and every sibling + # test is refused before reaching that call. Raising from the stub + # proves the guard did not fire *and* that execution got as far as + # the preflight. + with mock.patch.object( + cli, "_prepare_graded_run", + side_effect=SystemExit("error: preflight reached"), + ): + message = self._run("--harness", "claude-code") + self.assertNotIn("no staleness leg", message) + self.assertIn("preflight reached", message) + + def _pair(self, with_results, without_results): + return staleness.pair( + [_outcome("c", ok, attempt=i) for i, ok in enumerate(with_results, 1)], + [_outcome("c", ok, attempt=i) for i, ok in enumerate(without_results, 1)], + )[0] + + def test_the_counts_reach_the_comparison(self) -> None: + case = self._pair([True, True, True], [False, False, False]) + self.assertEqual((case.passes_with, case.attempts_with), (3, 3)) + self.assertEqual((case.passes_without, case.attempts_without), (0, 3)) + + def test_one_attempt_each_is_the_rule_that_was_there_before(self) -> None: + # The property that makes this safe to add: at one attempt a side, + # every verdict is the one the single-pair rule gave. + self.assertEqual(self._pair([True], [False]).verdict, "earns_it") + self.assertEqual(self._pair([True], [True]).verdict, "no_signal") + self.assertEqual(self._pair([False], [True]).verdict, "harmful") + self.assertEqual(self._pair([False], [False]).verdict, "no_signal") + + def test_a_side_passes_when_any_attempt_did(self) -> None: + # "Any" rather than "all". A skill that works two times in three has + # done something, and counting it as a side that failed would say it + # had not. + case = self._pair([False, True, False], [False, False, False]) + self.assertEqual(case.verdict, "earns_it") + self.assertEqual(case.passes_with, 1) + + def test_two_flaky_sides_are_no_signal_and_say_so_in_the_counts(self) -> None: + # The verdict cannot tell this from a clean `no_signal`, which is why + # the counts are printed beside it. + case = self._pair([True, False], [False, True]) + self.assertEqual(case.verdict, "no_signal") + self.assertEqual((case.passes_with, case.passes_without), (1, 1)) + + def test_a_baseline_that_managed_it_once_takes_the_case_to_no_signal(self) -> None: + # The rule written down, because nothing asserted it and so the docs + # and the code were free to disagree about it -- `docs/usage.md` and + # `_passes` both said `2/3` against `1/3` was `earns_it`. It is not. + # + # What follows from "any attempt passed" is that a higher attempt + # count is a *higher bar*: `earns_it` at three needs the baseline to + # have failed all three times. So a verdict compares with one taken + # at the same count and not with one taken at another. + self.assertEqual( + self._pair([True, True, False], [False, True, False]).verdict, "no_signal" + ) + self.assertEqual( + self._pair([True, True, True], [False, False, False]).verdict, "earns_it" + ) + self.assertEqual( + self._pair([True, True, True], [True, False, False]).verdict, "no_signal" + ) + + def test_an_errored_attempt_leaves_the_others_their_say(self) -> None: + # One dropped connection of three is not a side that measured + # nothing. The denominator shrinks; the verdict stands on the rest. + case = staleness.pair( + [_outcome("c", True, attempt=1), + _outcome("c", False, attempt=2, error="API Error: 504"), + _outcome("c", True, attempt=3)], + [_outcome("c", False, attempt=i) for i in (1, 2, 3)], + )[0] + self.assertEqual(case.verdict, "earns_it") + self.assertEqual((case.passes_with, case.attempts_with), (2, 2)) + self.assertEqual(case.errored_with, 1) + + def test_a_side_whose_every_attempt_errored_is_ungraded(self) -> None: + case = staleness.pair( + [_outcome("c", False, attempt=i, error="boom") for i in (1, 2)], + [_outcome("c", False, attempt=i) for i in (1, 2)], + )[0] + self.assertEqual(case.verdict, "ungraded") + self.assertEqual((case.attempts_with, case.errored_with), (0, 2)) + + +class TestContaminationVoidsOnTheAttemptTheVerdictRestsOn(unittest.TestCase): + """A dirty baseline voids what it actually refutes, and no more. + + `no_signal` and `harmful` claim the baseline *managed* something unaided. + An attempt it failed cannot be the attempt they rest on, so contamination + there refutes nothing. With one attempt a side the two readings agree; + they part company once a side can be dirty on one attempt and pass on + another. + """ + + def _baseline(self, passed, contaminated, attempt=1): + outcome = _outcome("c", passed, attempt=attempt) + outcome.contaminated = contaminated + return outcome + + def test_a_dirty_baseline_that_passed_voids_no_signal(self) -> None: + case = staleness.pair( + [_outcome("c", True)], [self._baseline(True, "another skill fired: beta")] + )[0] + self.assertEqual(case.verdict, "ungraded") + self.assertTrue(case.baseline_dirty_on_pass) + + def test_a_dirty_baseline_that_failed_leaves_earns_it_standing(self) -> None: + # It fails all the harder without what it should not have had, so + # voiding it would throw away a conclusion the run is entitled to. + case = staleness.pair( + [_outcome("c", True)], [self._baseline(False, "another skill fired: beta")] + )[0] + self.assertEqual(case.verdict, "earns_it") + self.assertFalse(case.baseline_dirty_on_pass) + self.assertTrue(case.baseline_contaminated, "the finding is still reported") + + def test_a_clean_passing_attempt_is_not_voided_by_a_dirty_failed_one(self) -> None: + case = staleness.pair( + [_outcome("c", True, attempt=1)], + [self._baseline(False, "another skill fired: beta", attempt=1), + self._baseline(True, None, attempt=2)], + )[0] + self.assertEqual(case.verdict, "no_signal") + self.assertFalse(case.baseline_dirty_on_pass) + + def test_a_dirty_baseline_that_passed_voids_harmful_too(self) -> None: + # The other half of `VOIDED_BY_CONTAMINATION`, and the one the module + # calls rare and worth stopping for. Nothing asserted it, so dropping + # `harmful` from that tuple left the whole suite green. + case = staleness.pair( + [_outcome("c", False)], + [self._baseline(True, "read the skill's own files (x)")], + )[0] + self.assertEqual(case.verdict, "ungraded") + + def test_a_verdict_resting_on_a_clean_attempt_is_not_called_a_failure(self) -> None: + # The paragraph that explains which verdicts survive contamination + # told the reader they rest on the baseline having *failed*. For this + # case the baseline passed: it was dirty on an attempt it failed and + # clean on the one the verdict rests on. + comparisons = staleness.pair( + [_outcome("c", True, attempt=1)], + [self._baseline(False, "another skill fired: beta", attempt=1), + self._baseline(True, None, attempt=2)], + ) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("rest on a clean attempt", report) + self.assertNotIn("rest on the baseline *failing*", report) + + def test_a_baseline_that_passed_nothing_is_not_told_it_passed(self) -> None: + # The same paragraph, the other way wrong. `no_signal` covers both + # sides *failing* as well as both passing, and `dirty_on_pass` fires + # only on an attempt that passed -- so a baseline dirty on the one + # attempt it had, and failing it, is never voided and landed in the + # group whose sentence says the baseline "passed on an attempt that + # was clean". It passed on no attempt at all. One side one attempt: + # `--attempts` is not needed to reach this. + comparisons = staleness.pair( + [_outcome("c", False)], + [self._baseline(False, "another skill fired: beta")], + ) + self.assertEqual(comparisons[0].verdict, "no_signal") + self.assertEqual(comparisons[0].passes_without, 0) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertNotIn("rest on a clean attempt", report) + self.assertIn("passed nothing on any attempt", report) + + +class TestEarningItIsReportedWithWhetherTheSkillFired(unittest.TestCase): + """Installing a skill does not make an agent use it. + + Firing used to be a verdict of its own (`unattributed`). It is now a flag + on the comparison and a note in the report, which is a weaker statement + than a verdict and the honest one: the outcome did differ, so the case is + `earns_it`; what is unknown is whether the skill is why. The distinction + still has to survive, or a case the skill never touched is credited to it + in the one number this command is read for. + """ + + def _pair(self, activated): + with_skill = _checked("c", True, [_check("ran the CLI", True)]) + with_skill.activated = activated + return staleness.pair( + [with_skill], [_checked("c", False, [_check("ran the CLI", False)])] + ) + + def test_the_firing_is_carried_onto_the_comparison(self) -> None: + self.assertTrue(self._pair("rocm-doctor")[0].fired_with) + self.assertFalse(self._pair(None)[0].fired_with) + + def test_the_order_of_two_passing_attempts_does_not_decide_it(self) -> None: + # Read off `_decisive`'s pick -- the *first* passing attempt -- the + # same two runs in the other order came out as not having fired. + # Which of two attempts happened to run first is not a property of + # the skill. + for order in ((None, "rocm-doctor"), ("rocm-doctor", None)): + with self.subTest(order=order): + case = staleness.pair( + [_checked("c", True, [_check("ran the CLI", True)], + attempt=i, activated=fired) + for i, fired in enumerate(order, 1)], + [_checked("c", False, [_check("ran the CLI", False)])], + )[0] + self.assertEqual(case.verdict, "earns_it") + self.assertTrue(case.fired_with) + + def test_an_attempt_that_fired_and_failed_does_not_count_as_firing(self) -> None: + # The `_met(a) and` half of the guard. The skill fired on an attempt + # the agent *failed*, and the attempt that passed never opened it. + # "The skill fired" and "the skill moved the outcome" are the two + # claims this keeps apart, and this is where they come apart hardest. + case = staleness.pair( + [_checked("c", False, [_check("x", False)], attempt=1, activated="s"), + _checked("c", True, [_check("x", True)], attempt=2, activated=None)], + [_checked("c", False, [_check("x", False)], attempt=1, activated=None)], + )[0] + self.assertEqual(case.verdict, "earns_it") + self.assertFalse(case.fired_with) + + def test_a_case_that_moved_without_firing_is_still_counted(self) -> None: + # The change from the longer taxonomy, stated outright. The outcome + # differed, so it is `earns_it` and it is in the share. What is not + # known is whether the skill is why, which the count below carries. + totals = staleness.summarize( + self._pair(None), {"model": "m", "effort": "e"} + )["totals"] + self.assertEqual(totals["earns_it"], 1) + self.assertEqual(totals["earns_it_unfired"], 1) + self.assertEqual(totals["discriminating"], 1.0) + + def test_a_case_that_moved_after_firing_is_not_flagged(self) -> None: + totals = staleness.summarize( + self._pair("rocm-doctor"), {"model": "m", "effort": "e"} + )["totals"] + self.assertEqual(totals["earns_it"], 1) + self.assertEqual(totals["earns_it_unfired"], 0) + + def test_the_reader_is_warned_before_believing_the_number(self) -> None: + # The share now counts these, so the warning is what keeps it honest. + # It has to come before the reader reaches the case table. + summary = staleness.summarize(self._pair(None), {"model": "m", "effort": "e"}) + report = staleness.render_markdown(summary) + self.assertIn("passed without the skill ever firing", report) + self.assertLess( + report.index("without the skill ever firing"), + report.index("### Cases worth a look"), + ) + + def test_a_clean_run_says_nothing_about_firing(self) -> None: + # No note when every earns_it fired: an unconditional one would be + # noise, and noise is what gets skipped on the run that matters. + report = staleness.render_markdown( + staleness.summarize(self._pair("rocm-doctor"), {"model": "m", "effort": "e"}) + ) + self.assertNotIn("without the skill ever firing", report) + + +class TestTheBaselineRunIsTheSameRunWithoutTheSkill(unittest.TestCase): + """The flag has to reach the driver, or the two halves are not comparable.""" + + def _staged(self, *, install: bool) -> list: + """Whether the solver reached `install_skill`, for one value of the flag. + + Driven rather than read. The earlier version of this asserted that the + signature had an `install` parameter defaulting to True, which says + nothing about whether the flag reaches the staging step -- and the + staging step is the entire difference between the two halves. + + The CLI call after it is cut off by giving it no output to parse, which + is the first thing the solver refuses: staging has already happened or + not by then. + """ + import asyncio + + staged: list[Path] = [] + workspace = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, workspace, ignore_errors=True) + + async def no_workspace(): + return workspace + + async def no_output(*args, **kwargs): + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + + solve = engine_no_sandbox.claude_code_no_sandbox( + None, None, Path("alpha"), install=install + ) + state = types.SimpleNamespace(input_text="p", messages=[], store=None) + with mock.patch.object(engine_no_sandbox, "_workspace", no_workspace), \ + mock.patch.object( + engine_no_sandbox, "install_skill", + lambda d, w, skills_dir=None: staged.append(d), + ), \ + mock.patch("inspect_ai.util.subprocess", no_output): + with self.assertRaises(RuntimeError): + asyncio.run(solve(state, None)) + return staged + + def test_the_host_driver_stages_nothing_when_asked_not_to(self) -> None: + self.assertEqual(self._staged(install=False), []) + + def test_the_host_driver_stages_the_skill_when_asked_to(self) -> None: + # The other half: a driver that staged nothing ever would pass the + # test above while making both halves the baseline. + self.assertEqual(self._staged(install=True), [Path("alpha")]) + + def test_the_task_name_says_which_half_it_is(self) -> None: + # Both halves write into one log directory; a name that did not say + # would leave the two indistinguishable on disk. Asserted on the names + # the two calls actually produce, rather than on the source containing + # the suffix somewhere. + repo = Repo(self) + repo.skill("alpha", dataset=tier0_dataset("alpha")) + repo.activate() + cases = [c for c in datasets.load_all_cases() if c.skill == "alpha"][:1] + + graded = engine_behavioral.build_task( + "alpha", cases, "mockllm/model", solver_factory=lambda d: None + ) + baseline = engine_behavioral.build_task( + "alpha", cases, "mockllm/model", + solver_factory=lambda d: None, baseline=True, + ) + self.assertFalse(graded.name.endswith("-baseline")) + self.assertTrue(baseline.name.endswith("-baseline")) + self.assertIn("graded", graded.tags) + self.assertIn("baseline", baseline.tags) + + +class TestTheDetectorKeepsItsTwoAnswersApart(unittest.TestCase): + """`_inspect_room` answers two questions, and they must not cross. + + `fired` is `None` or exactly the skill under test -- `detect_activation` is + called with `skills=[skill]`, so there is nothing else it can legitimately + be. When the two answers shared one `if/elif`, a second `other:` hit + arriving after `why` was already set fell through to the `elif` and was + recorded as the skill having fired. + + That string then reads as truthy everywhere downstream: `activated_with` + on the `Comparison`, and the `earns_it` / `unattributed` split, which + turns on exactly that field. So a case was credited to a skill that never + fired on exactly the contaminated runs that most need the opposite, and + the marker landed verbatim in both JSON reports. + """ + + def _sample(self, calls): + return _FakeSample(messages=[_FakeMessage("m", calls)], events=[]) + + def _call(self, name, arguments=None): + call = _FakeCall("toolu_1", function=name) + call.arguments = arguments or {} + return call + + def _inspect(self, sample, skill="alpha"): + from skillscope.engine import routing as routing_engine + + # Marks for the skill, so the room is knowable and `why` reports + # what this class is about rather than `UNKNOWN_ROOM`. No case here + # names a path, so nothing can match them. + with mock.patch.object( + engine_behavioral, "_skill_marks", + lambda name: (f"/nonexistent/skills/{name}", f".claude/skills/{name}"), + ): + return engine_behavioral._inspect_room(routing_engine, sample, skill) + + def test_a_contamination_marker_is_never_reported_as_the_skill(self) -> None: + # Two `other:` hits. The first sets `why`; the second used to find it + # already set, fail the first condition, and land in `fired`. + sample = self._sample([ + self._call("Skill", {"command": "beta"}), + self._call("Skill", {"command": "gamma"}), + ]) + fired, why = self._inspect(sample) + self.assertIsNone(fired) + self.assertIsNotNone(why) + self.assertIn("another skill fired", why) + + def test_the_skill_under_test_is_still_reported_when_it_fires(self) -> None: + # The other half: a fix that reports nothing would pass the test above. + sample = self._sample([self._call("Skill", {"command": "alpha"})]) + fired, why = self._inspect(sample) + self.assertEqual(fired, "alpha") + self.assertIsNone(why) + + def test_a_marker_cannot_reach_the_report_as_an_activation(self) -> None: + # The consequence, asserted where it is read: a truthy garbage string + # read as an activation suppresses the note this totals. + sample = self._sample([ + self._call("Skill", {"command": "beta"}), + self._call("Skill", {"command": "gamma"}), + ]) + fired, _ = self._inspect(sample) + graded = behavior.BehaviorOutcome( + id="c", skill="alpha", prompt="p", passed=True, elapsed_s=0.0, + checks=[_check("do it", True)], activated=fired, + ) + without = _checked("c", False, [_check("do it", False)], skill="alpha") + totals = staleness.summarize( + staleness.pair([graded], [without]), {"model": "m", "effort": "e"} + )["totals"] + self.assertEqual(totals["earns_it_unfired"], 1) + self.assertEqual(totals["earns_it"], 1) + + +class TestTheSandboxedBaselineIsGivenNoSkill(unittest.TestCase): + """The baseline half of the `claude-code` leg, built and read. + + `install` was asserted by grepping this function's source, so the whole + of what it controls -- the staging solver, the agent's own `skills` + argument, the task name, the metadata and the tags -- could be wired to + the graded value with the suite green. That failure is silent in the + worst way: the baseline becomes a second graded run, every case comes + out `no_signal`, and the report says the skill earns nothing. + + `inspect_swe` is not installed in the unit job, so `claude_code` is + stubbed the way `TestEveryLegThinksAsHardAsItWasTold` stubs it. The stub + returns a real solver so `chain` still builds. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + # The behavioural dataset, not `tier0_dataset`: `verify.run` skips a + # skill with no case asserting a behaviour, so the graded leg would + # never be built. + self.repo.skill("run", dataset=_behavioral_dataset("run")) + self.repo.activate() + self.cases = [c for c in datasets.load_all_cases() + if c.skill == "run" and c.has_behavior][:1] + + def _task(self, install: bool): + seen: dict = {} + fake = types.ModuleType("inspect_swe") + fake.claude_code = lambda **k: seen.update(agent=k) or engine_verify._ensure_workdir() + staging = lambda skills: seen.update(staged=skills) or engine_verify._ensure_workdir() # noqa: E731 + with mock.patch.dict(sys.modules, {"inspect_swe": fake}), \ + mock.patch.object(engine_verify, "complete_skills_solver", staging): + task = engine_verify.build_task( + "run", self.cases, "mockllm/model", effort="high", install=install, + ) + return task, seen + + def test_the_baseline_stages_nothing_and_is_told_nothing(self) -> None: + task, seen = self._task(install=False) + self.assertEqual(seen["staged"], {}, "the baseline staged the skill") + self.assertEqual(seen["agent"]["skills"], [], "the agent was given the skill") + self.assertIn("baseline", task.tags) + self.assertTrue(task.name.endswith("-baseline")) + + def test_the_graded_half_gets_the_skill_both_ways(self) -> None: + # The control. Each assertion above reads the same on both halves + # unless the other half is built too. + task, seen = self._task(install=True) + self.assertEqual(list(seen["staged"]), ["run"]) + self.assertEqual(len(seen["agent"]["skills"]), 1) + self.assertIn("graded", task.tags) + self.assertFalse(task.name.endswith("-baseline")) + + def test_the_runner_hands_its_half_down_to_the_task(self) -> None: + # The join: `run(install=False)` has to reach `build_task`. Dropping + # it there builds a graded task from a baseline run, which no + # assertion above can see. + seen: dict = {} + with mock.patch.object(engine_verify, "require", lambda: None), \ + mock.patch.object(engine_verify.sandbox_spec, "require_provider", + lambda: None), \ + mock.patch.object(engine_verify, "build_task", + lambda *a, **k: seen.update(k) or "task"), \ + mock.patch.object(engine_verify.behavioral, "run_eval", + lambda ev, task, **kw: seen.update(kw) or []): + engine_verify.run( + ["run"], self.cases, "mockllm/model", "high", + install=False, attempts=3, + ) + self.assertFalse(seen["install"]) + self.assertEqual(seen["epochs"], 3) + + +class TestContaminationIsJudgedOnTheSkillsRealPath(unittest.TestCase): + """A bare skill name is not evidence that the skill's files were read. + + `structure.py` allows a single-word name and sets no minimum length, and + single-word names are the norm -- Anthropic ship `pdf`, `docx`, `pptx`. The + heuristic matched `/{skill}/` as a substring of every tool call's + serialised arguments, so a skill named `run` was reported as contaminated + by any agent that touched `/run/user/1000`, and `docs`, `test`, `tools` or + `scripts` by any ordinary path. + + The cost of that false positive is the highest there is here: + `_voided_if_dirty` converts `no_signal` and `harmful` -- the two verdicts + the command exists to surface -- into `ungraded`, and the report states the + contamination as fact. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill("run", dataset=tier0_dataset("run")) + self.repo.activate() + + def _why(self, arguments, skill="run"): + from skillscope.engine import routing as routing_engine + + call = _FakeCall("toolu_1", function="Read") + call.arguments = arguments + sample = _FakeSample(messages=[_FakeMessage("m", [call])], events=[]) + return engine_behavioral._inspect_room(routing_engine, sample, skill)[1] + + def test_an_unrelated_path_sharing_the_skills_name_is_not_contamination(self) -> None: + self.assertIsNone(self._why({"file_path": "/run/user/1000/bus"})) + + def test_a_bash_command_mentioning_the_name_is_not_contamination(self) -> None: + self.assertIsNone( + self._why({"command": "ls /run/systemd && mkdir -p build/run/out"}) + ) + + def test_the_skills_own_directory_still_counts(self) -> None: + own = str(config.active().skill_path("run") / "SKILL.md") + self.assertIn("read the skill's own files", self._why({"file_path": own})) + + def test_the_staged_copy_still_counts(self) -> None: + staged = "/tmp/ws-1234/.claude/skills/run/reference.md" + self.assertIn("read the skill's own files", self._why({"file_path": staged})) + + def test_a_bash_command_naming_the_real_directory_still_counts(self) -> None: + own = str(config.active().skill_path("run")) + self.assertIn( + "read the skill's own files", self._why({"command": f"cat {own}/SKILL.md"}) + ) + + def test_the_harness_staging_directory_still_counts(self) -> None: + self.assertIn( + "read the skill's own files", + self._why({"file_path": "/opt/_bundled_skills/run/SKILL.md"}), + ) + + def test_a_sibling_skill_whose_name_extends_this_one_is_not_this_one(self) -> None: + # The false positive that survived the first fix in the shell branch, + # because the staged mark carried no trailing boundary there while + # the structured branch used `_under`. `run-extra` is its own skill. + self.assertIsNone( + self._why({"command": "cat /home/u/.claude/skills/run-extra/SKILL.md"}) + ) + self.assertIsNone( + self._why({"file_path": "/home/u/.claude/skills/run-extra/SKILL.md"}) + ) + + def test_a_name_the_skills_own_name_is_a_prefix_of_is_not_a_hit(self) -> None: + own = str(config.active().skill_path("run")) + self.assertIsNone(self._why({"file_path": f"{own}.bak/SKILL.md"})) + + def test_the_staged_copy_is_found_by_the_relative_name(self) -> None: + # `no_sandbox` runs the agent with `cwd=workspace`, so this is the + # natural shell form. A mark anchored with a leading separator missed + # it, and a missed contamination is the worse direction: the baseline + # is then reported clean, as fact. + self.assertIn( + "read the skill's own files", + self._why({"command": "cat .claude/skills/run/SKILL.md"}), + ) + + def test_an_argument_the_field_list_never_named_is_still_read(self) -> None: + # `pattern` is Glob's primary argument and is often its only one, so + # the old fixed field list made the call invisible. The tool surface + # is open, so the list was always going to drift behind it. + own = str(config.active().skill_path("run")) + self.assertIn( + "read the skill's own files", self._why({"pattern": f"{own}/**/*.md"}) + ) + + def test_a_tool_naming_its_arguments_anything_is_still_read(self) -> None: + # An MCP tool names its arguments whatever it likes. + own = str(config.active().skill_path("run")) + self.assertIn( + "read the skill's own files", self._why({"some_mcp_arg": f"{own}/x.md"}) + ) + + def test_a_path_nested_inside_a_structured_argument_is_still_read(self) -> None: + own = str(config.active().skill_path("run")) + self.assertIn( + "read the skill's own files", + self._why({"edits": [{"file": f"{own}/SKILL.md"}]}), + ) + + def test_a_skill_with_no_directory_does_not_take_the_command_down(self) -> None: + # `config.skill_path` raises `SystemExit`, which `except Exception` + # does not catch -- so rebuilding somebody else's logs killed the + # process rather than reporting what it could. + why = self._why({"file_path": "/tmp/x"}, skill="not-a-skill") + self.assertIsNotNone(why) + + def test_a_skill_with_no_directory_is_an_unknown_room_not_a_clean_one(self) -> None: + # It used to answer `None` here, which reads as "the baseline was + # clean" -- a verdict reached by not looking. The mark it dropped is + # the only one an absolute path can hit, so every tool call that + # reached the skill by its real path came back clean on any machine + # without the skill checked out. + why = self._why({"file_path": "/tmp/x"}, skill="not-a-skill") + self.assertEqual(why, engine_behavioral.UNKNOWN_ROOM) + + +class TestAWalkThatDidNotFinishCannotVouchForTheRoom(unittest.TestCase): + """An interrupted walk inspected only the calls it got to. + + `_inspect_room` computes `unknown` once, before the walk, from whether + there were marks to check against. That answers "could the read half of + the check run at all", not "did the check actually run", and the `except` + branch used to return the same expression a completed walk returns. So a + walk that died on its second call reported the room clean, as fact, with + the contamination sitting on the third call unread. + + `_activation_event` reaches `call.function` with no `getattr` guard, + unlike every other accessor on this path, so an unexpected call shape is + the realistic way in -- and the offline reader this round scoped out, + which will read logs this code did not write, is exactly where an + unexpected shape would come from. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill("run", dataset=tier0_dataset("run")) + self.repo.activate() + + def test_a_walk_cut_short_is_an_unknown_room_not_a_clean_one(self) -> None: + from skillscope.engine import routing as routing_engine + + class _Unwalkable: + id = "toolu_2" + arguments: dict = {} + + @property + def function(self): + raise AttributeError("a shape this code did not write") + + innocuous = _FakeCall("toolu_1", function="Read") + innocuous.arguments = {"file_path": "/tmp/notes.md"} + # Never reached, and the whole point: the evidence the walk would + # have found had it got there. + unread = _FakeCall("toolu_3", function="Read") + own = config.active().skill_path("run") / "SKILL.md" + unread.arguments = {"file_path": str(own)} + + sample = _FakeSample( + messages=[_FakeMessage("m", [innocuous, _Unwalkable(), unread])], + events=[], + ) + _fired, why = engine_behavioral._inspect_room(routing_engine, sample, "run") + self.assertEqual(why, engine_behavioral.UNKNOWN_ROOM) + + +class TestABaselineThatNeverFinishedIsNotABaselineThatWasFast(unittest.TestCase): + """The per-case column, and the one way it can invert its own finding. + + A side whose attempts all errored has no time to report. Writing that as + `0s` beside a graded half's `2m23s` says the baseline was instant, which + is the exact opposite of the case this command was built for: a baseline + that ran for half an hour and never concluded. + """ + + @staticmethod + def _timed(case_id, passed, seconds, *, attempt=1, error=None): + out = _outcome(case_id, passed, attempt=attempt, error=error) + out.elapsed_s = seconds + return out + + def test_a_side_is_the_sum_of_the_attempts_that_answered(self) -> None: + # Summed, so the column reconciles with the per-half total above it. + case = staleness.pair( + [self._timed("c", True, 10.0, attempt=1), + self._timed("c", False, 5.0, attempt=2)], + [self._timed("c", False, 2.0)], + )[0] + self.assertEqual(case.seconds_with, 15.0) + + def test_an_attempt_that_errored_contributes_no_time(self) -> None: + case = staleness.pair( + [self._timed("c", True, 10.0, attempt=1), + self._timed("c", False, 99.0, attempt=2, error="API Error: 504")], + [self._timed("c", False, 2.0)], + )[0] + self.assertEqual(case.seconds_with, 10.0) + + def test_a_side_with_no_time_at_all_is_none_not_zero(self) -> None: + case = staleness.pair( + [self._timed("c", True, 10.0)], + [self._timed("c", False, 0.0, error="API Error: 504")], + )[0] + self.assertIsNone( + case.seconds_without, + "zero seconds and no measurement are different findings here", + ) + + def test_a_side_that_never_finished_says_so_in_the_report(self) -> None: + comparisons = staleness.pair( + [self._timed("c", True, 143.2)], + [self._timed("c", False, 0.0, error="deadline stopped the run")], + ) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("did not finish", report) + self.assertNotIn("0s", report) + + def test_a_case_missing_from_a_side_is_not_run_rather_than_instant(self) -> None: + comparisons = staleness.pair([self._timed("c", True, 10.0)], []) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("not run", report) + + def test_the_times_reach_the_case_table(self) -> None: + comparisons = staleness.pair( + [self._timed("c", True, 143.2)], [self._timed("c", True, 1680.4)] + ) + report = staleness.render_markdown(staleness.summarize(comparisons, {})) + self.assertIn("2m23s", report) + self.assertIn("28m00s", report) + + +class TestTheTwoHalvesAreBilledSeparately(unittest.TestCase): + """One global accumulator, two halves, and a trap in how it is read. + + `usage.snapshot()` returns `_current` itself rather than a copy, so the + obvious way to measure a half -- hold a snapshot, subtract it afterwards + -- subtracts the totals from themselves and reports both halves as having + spent nothing. That failure produces plausible zeros rather than an error, + which is the kind nobody catches by reading a report. + + So the scope swaps the global instead, and these pin both halves of that: + each leg sees only its own, and the command total afterwards is still the + sum, which is what keeps every other report's meta unchanged. + """ + + def setUp(self) -> None: + usage.reset() + self.addCleanup(usage.reset) + + def test_each_half_holds_only_its_own_spend(self) -> None: + with usage.leg("graded") as graded: + usage.record(input_tokens=10, output_tokens=1, calls=1) + with usage.leg("baseline") as baseline: + usage.record(input_tokens=100, output_tokens=20, calls=7) + self.assertEqual((graded.usage.input_tokens, graded.usage.calls), (10, 1)) + self.assertEqual((baseline.usage.input_tokens, baseline.usage.calls), (100, 7)) + + def test_the_command_total_is_still_the_sum(self) -> None: + # The no-regression property. `behavioral` and `routing` splice + # `usage.snapshot()` into their own meta and must not notice this. + with usage.leg("graded"): + usage.record(input_tokens=10, calls=1) + with usage.leg("baseline"): + usage.record(input_tokens=100, calls=7) + self.assertEqual(usage.snapshot().input_tokens, 110) + self.assertEqual(usage.snapshot().calls, 8) + + def test_a_snapshot_taken_inside_a_leg_is_that_leg(self) -> None: + # The trap, made explicit. A caller holding this reference and + # subtracting it later gets zero, which is why the scope swaps. + with usage.leg("graded") as graded: + usage.record(input_tokens=10, calls=1) + self.assertIs(usage.snapshot(), graded.usage) + + def test_an_unknown_cost_does_not_fold_back_as_zero(self) -> None: + with usage.leg("graded"): + usage.record(input_tokens=10, calls=1) + self.assertIsNone(usage.snapshot().cost_usd) + + def test_a_half_that_raised_still_hands_the_global_back(self) -> None: + # The `finally`, which nothing pinned: every other test here reads + # the leg's own accumulator, which is correct whether or not the + # global was restored. Without it a raising half leaves `_current` + # pointing at that half for the rest of the process, so the second + # half's spend lands in the first half's figures and the command + # total silently stops being the sum. + # Asserted by identity, not by a total. Both ways the number comes + # out the same: the fold-back adds exactly what the leaked + # accumulator already holds, so `snapshot().input_tokens` is 15 + # whether the global was restored or left pointing at the half. What + # differs is *which object* it is. + held = [] + with self.assertRaises(RuntimeError): + with usage.leg("graded") as graded: + held.append(graded) + usage.record(input_tokens=10, calls=1) + raise RuntimeError("the engine fell over") + self.assertIsNot(usage.snapshot(), held[0].usage, + "the global was left pointing at the half that raised") + usage.record(input_tokens=5, calls=1) + self.assertEqual(usage.snapshot().input_tokens, 15) + + def test_the_wall_clock_and_the_rest_fold_back_too(self) -> None: + # `wall_time_s` is set in the same `finally`, and the report's floor + # marker and spend table both read it. + with usage.leg("graded") as graded: + usage.record(input_tokens=10, output_tokens=3, calls=1) + self.assertGreaterEqual(graded.wall_time_s, 0.0) + self.assertEqual(graded.usage.output_tokens, 3) + self.assertEqual(usage.snapshot().output_tokens, 3) + + def test_an_engine_recording_from_a_log_lands_in_the_enclosing_half(self) -> None: + # How the real thing records: `stats.record_log` is called deep inside + # the engine, never at the leg boundary, and reaches the global by + # name at call time. That is what the swap relies on. + log = types.SimpleNamespace( + stats=types.SimpleNamespace( + model_usage={"m": types.SimpleNamespace( + input_tokens=42, output_tokens=5, total_cost=None + )} + ), + samples=[], + ) + with usage.leg("baseline") as baseline: + engine_stats.record_log(log) + self.assertEqual(baseline.usage.input_tokens, 42) + + def test_a_half_that_raises_still_folds_back(self) -> None: + # The deadline stops the baseline by unwinding through here. A scope + # that only folded back on success would lose the stopped half's + # spend -- the half most worth knowing the cost of. + with contextlib.suppress(RuntimeError): + with usage.leg("baseline"): + usage.record(input_tokens=100, calls=7) + raise RuntimeError("stopped") + self.assertEqual(usage.snapshot().input_tokens, 100) + + +class TestTheReportSaysWhatEachHalfCost(unittest.TestCase): + """The signal the first version of this command measured and threw away. + + A graded half finished in 2m23s while the baseline was still going half + an hour later, and the report said nothing about it: both halves' cases + passed, so every verdict read `no_signal`. The verdicts cannot express + this, which is why it is said above them. + """ + + @staticmethod + def _summary(*, finished=True, baseline_s=1680.4, cost=None, meta=None): + halves = [ + {"label": "graded", "wall_time_s": 143.2, "finished": True, + "model_calls": 37, "total_tokens": 184220}, + {"label": "baseline", "wall_time_s": baseline_s, + "finished": finished, "model_calls": 214, "total_tokens": 1044901}, + ] + if cost is not None: + for half, value in zip(halves, cost): + half["cost_usd"] = value + return staleness.summarize( + staleness.pair([_outcome("a", True)], [_outcome("a", True)]), + {"legs": halves, **(meta or {})}, + ) + + def _report(self, **kw) -> str: + return staleness.render_markdown(self._summary(**kw)) + + def test_a_baseline_that_was_stopped_is_not_a_baseline_that_was_quick(self) -> None: + report = self._report(finished=False) + self.assertIn("The baseline never finished", report) + self.assertIn("28m00s+", report) + self.assertIn("is a floor", report) + + def test_a_baseline_that_finished_is_compared_as_a_ratio(self) -> None: + report = self._report() + self.assertIn("11.7x the graded half", report) + self.assertNotIn("never finished", report) + + def test_a_skill_that_costs_time_is_said_so_rather_than_hidden(self) -> None: + # A ratio under one is a finding about the skill, not a number to + # suppress because it is unflattering. + report = self._report(baseline_s=30.0) + self.assertIn("The graded half was the slower of the two", report) + + def test_the_spend_is_above_the_verdicts(self) -> None: + report = self._report() + self.assertLess(report.index("What each half cost"), report.index("| Verdict |")) + + def test_an_unknown_cost_reads_as_unknown_and_says_why(self) -> None: + report = self._report() + self.assertIn("| n/a |", report) + self.assertIn("missing cost is a missing number, not a zero", report) + + def test_a_known_cost_is_printed_without_the_caveat(self) -> None: + report = self._report(cost=(0.14, 1.80)) + self.assertIn("0.14", report) + self.assertNotIn("missing cost is a missing number", report) + + def test_running_cases_at_once_is_disclosed_against_the_clock(self) -> None: + # Wall time stops meaning agent time the moment cases overlap, and + # this table is read as though it did. + self.assertIn( + "ran at once on each half", self._report(meta={"concurrency": 4}) + ) + self.assertNotIn("ran at once on each half", self._report()) + + def test_a_summary_with_no_halves_still_renders(self) -> None: + # `summarize` is called by tests and by anything holding only + # comparisons; the spend section is additive and must not be assumed. + report = staleness.render_markdown( + staleness.summarize(staleness.pair([_outcome("a", True)], [_outcome("a", False)]), {}) + ) + self.assertNotIn("What each half cost", report) + + +class TestBothHalvesRunAndOnlyTheInstallFlagDiffers(unittest.TestCase): + """The command's whole measurement is that the two runs differ in one way. + + Called rather than read. Every earlier assertion about `cmd_staleness` was + a substring check against its source, and `cli._run_behavioral` was never + invoked by a test at all. + """ + + def test_the_cases_are_run_twice(self) -> None: + with _staleness_harness(self) as harness: + self.assertEqual(harness.run(), 0) + self.assertEqual(len(harness.calls), 2) + + def test_the_two_halves_disagree_about_installing(self) -> None: + # The regression this exists for: two calls with the same `install` + # compare a run against itself and report `no_signal` for everything. + with _staleness_harness(self) as harness: + harness.run() + self.assertEqual([c["install"] for c in harness.calls], [True, False]) + + def test_both_halves_grade_the_same_cases(self) -> None: + with _staleness_harness(self) as harness: + harness.run() + self.assertEqual(harness.calls[0]["cases"], harness.calls[1]["cases"]) + self.assertTrue(harness.calls[0]["cases"]) + + def test_the_half_with_the_skill_is_the_one_credited(self) -> None: + # The orientation of the whole command. The harness's graded half + # passes and its baseline fails, so every case must be `earns_it`; + # swapping `pair`'s arguments turns all of them into `harmful`, and + # nothing caught that because both halves used to return no outcomes + # at all. + with _staleness_harness(self) as harness: + harness.run() + totals = harness.reports[0]["totals"] + self.assertTrue(totals["earns_it"], "the graded half passed and was not credited") + self.assertFalse(totals["harmful"]) + for case in harness.reports[0]["cases"]: + self.assertEqual(case["verdict"], "earns_it") + + def test_the_time_each_half_spent_is_not_swapped_either(self) -> None: + # The same orientation, on the column where a swap reads as the + # finding this command exists to report: the baseline is the slow + # half, so `seconds_without` must be the larger number. + with _staleness_harness(self) as harness: + harness.run() + case = harness.reports[0]["cases"][0] + self.assertGreater(case["seconds_without"], case["seconds_with"]) + + def test_the_baseline_half_is_given_a_config_dir_of_its_own(self) -> None: + # The dir the comment calls what the whole command rests on: without + # it the runner's own `~/.claude/skills` are registered beside the + # staged ones and the baseline is this machine's skills minus the one + # under test. The harness recorded it and nothing read it, so handing + # both halves the same dir -- or `None` -- was green. + # The isolation check reads the environment, which another test in + # the suite may have cleared, so it is answered here rather than left + # to whatever ran before this. Its own refusal is covered by + # `test_the_command_refuses_an_unisolatable_host_run`. + with _staleness_harness(self) as harness: + with mock.patch.object(cli.routing, "can_isolate_config", lambda: True): + harness.run(engine="claude-code-no-sandbox") + graded, baseline = harness.calls + self.assertIsNotNone(baseline["config_dir"], "the baseline shares the host's") + self.assertNotEqual( + graded["config_dir"], baseline["config_dir"], + "one dir between the halves is one room between them", + ) + + def test_the_two_halves_are_billed_apart_in_the_report(self) -> None: + # End to end, through the real `cmd_staleness`. The stub spends a + # different amount per half, so a scope that captured both halves + # together -- or captured neither -- shows up here as the two rows + # agreeing. + with _staleness_harness(self) as harness: + harness.run() + legs = harness.reports[0]["meta"]["legs"] + self.assertEqual([half["label"] for half in legs], ["graded", "baseline"]) + self.assertEqual(legs[0]["input_tokens"], 10) + self.assertEqual(legs[1]["input_tokens"], 100) + + def test_the_command_total_still_covers_both_halves(self) -> None: + with _staleness_harness(self) as harness: + harness.run() + meta = harness.reports[0]["meta"] + self.assertEqual(meta["input_tokens"], 110) + self.assertEqual( + meta["input_tokens"], + sum(half["input_tokens"] for half in meta["legs"]), + "the totals are the sum of the halves, or one of them is wrong", + ) + + def test_a_half_the_bound_stopped_is_marked_unfinished(self) -> None: + # `finished` is what the report's floor marker rests on. The graceful + # stage fires before the wall, so `interrupted` is the condition that + # catches a baseline inspect cancelled. + with _staleness_harness(self) as harness: + bound = deadline.Deadline(seconds=600) + bound.interrupted = True + with mock.patch.object(deadline, "active", lambda: bound): + harness.run() + legs = harness.reports[0]["meta"]["legs"] + self.assertFalse(legs[0]["finished"]) + self.assertFalse(legs[1]["finished"]) + + def test_a_baseline_that_never_started_is_not_called_the_slow_one(self) -> None: + # The state above, read. The two halves share one bound, so a graded + # half that spends all of it leaves the baseline none: every baseline + # case returns failed without an agent starting. `_spend_lead` read + # only the baseline's flag, so the headline said it "was still going + # 0s in" -- a half that never ran, reported as the slow exploring one + # this whole command exists to find. Asserting the flags alone left + # that green, which is why this asserts the sentence. + with _staleness_harness(self) as harness: + bound = deadline.Deadline(seconds=600) + bound.interrupted = True + with mock.patch.object(deadline, "active", lambda: bound): + harness.run() + report = staleness.render_markdown(harness.reports[0]) + self.assertIn("before the baseline could start", report) + self.assertNotIn("The baseline never finished", report) + self.assertNotIn("was still going 0s in", report) + + def test_a_baseline_the_bound_stopped_partway_is_still_a_floor(self) -> None: + # The control, and the sentence that must survive: the graded half + # finished, the baseline was stopped mid-run. That is the half-hour + # baseline the command was written for, and it is still said as one. + with _staleness_harness(self) as harness: + harness.run() + summary = harness.reports[0] + graded, baseline = summary["meta"]["legs"] + graded.update(finished=True, wall_time_s=143.0) + baseline.update(finished=False, wall_time_s=1800.0) + report = staleness.render_markdown(summary) + self.assertIn("The baseline never finished", report) + self.assertNotIn("before the baseline could start", report) + + def test_an_unhurried_run_marks_both_halves_finished(self) -> None: + with _staleness_harness(self) as harness: + harness.run() + self.assertTrue(all(half["finished"] for half in harness.reports[0]["meta"]["legs"])) + + def test_the_caps_that_bound_a_run_are_recorded_in_its_report(self) -> None: + # The report's own caveat about a partial run is driven off these + # keys, so dropping them silently turns a capped run into one that + # reads as whole. Recorded only when they bound: a `0` would read as + # a cap of nothing rather than as no cap. + with _staleness_harness(self) as harness: + harness.run(extra=["--max-cases", "2", "--concurrency", "4"]) + meta = harness.reports[0]["meta"] + self.assertEqual(meta["max_cases"], 2) + self.assertEqual(meta["concurrency"], 4) + + with _staleness_harness(self) as plain: + plain.run() + self.assertNotIn("max_cases", plain.reports[0]["meta"]) + self.assertNotIn("concurrency", plain.reports[0]["meta"]) + + def test_the_report_names_the_harness_like_every_other_one(self) -> None: + # `cmd_staleness` hand-wrote its meta, so a staleness report was the + # one graded report with no `harness` or `harness_version` in it, and + # with the engine spelled without `_engine_label`'s normalisation. + # A reader holding this beside a routing or behavioral report found + # two keys missing and a third in a different alphabet. + with _staleness_harness(self) as harness: + harness.run() + meta = harness.reports[0]["meta"] + self.assertEqual(meta["harness"], cli.harnesses.DEFAULT) + self.assertIn("harness_version", meta) + self.assertEqual(meta["engine"], "claude-code") + + def test_a_dataset_with_nothing_to_grade_runs_neither_half(self) -> None: + with _staleness_harness(self, dataset=tier0_dataset("alpha")) as harness: + self.assertEqual(harness.run(), 0) + self.assertEqual(harness.calls, []) + + +class TestAStalenessRunHonoursHooksTheSameWayBehavioralDoes(unittest.TestCase): + """It drives the same `build_task`, so it cannot skip what that refuses to. + + `_require_hook_support` guarded on `command != "behavioral"`, and + `_prepare_graded_run` passes `args.command` -- so under `staleness` the + guard returned before it checked anything. A skill whose `evals/hooks.py` + defines `check` or `setup_session` had them silently skipped, on both + halves, which is the exact "dangerous half" the function exists to + prevent: the case is graded as though the hook ran. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + self.repo.skill( + "alpha", + dataset=tier0_dataset("alpha"), + hooks="def check(run):\n return True\n", + ) + self.repo.activate() + + def _refused(self, command): + try: + cli._require_hook_support("claude-code", ["alpha"], command) + except SystemExit as caught: + return str(caught) + return None + + def test_a_staleness_run_is_refused_the_same_as_a_behavioral_one(self) -> None: + refusal = self._refused("staleness") + self.assertIsNotNone(refusal) + self.assertIn("check", refusal) + + def test_the_behavioral_refusal_is_unchanged(self) -> None: + self.assertIsNotNone(self._refused("behavioral")) + + def test_routing_is_still_exempt(self) -> None: + # It installs the skills, asks which one fired, and executes nothing. + self.assertIsNone(self._refused("routing")) + + +class TestTheSecondPassGetsABudgetOfItsOwn(unittest.TestCase): + """Two passes under one bound, and the baseline is the slower half. + + Measured on a two-case skill: the graded pass finished both cases in + 2m23s, and the baseline had not finished the second after 30 minutes. An + agent holding the skill goes straight to the work; an agent without it + explores. On the default 900s the baseline got ~637s of a documented + 1800s+ need, which makes truncation the expected outcome on the command's + own canonical example rather than a risk. + """ + + def _timeout(self, argv): + return cli.build_parser().parse_args(argv).timeout + + def test_staleness_defaults_to_twice_the_shared_bound(self) -> None: + self.assertEqual( + self._timeout(["staleness"]), 2 * deadline.DEFAULT_TIMEOUT_S + ) + + def test_the_single_pass_commands_are_unchanged(self) -> None: + self.assertEqual( + self._timeout(["behavioral"]), deadline.DEFAULT_TIMEOUT_S + ) + self.assertEqual(self._timeout(["routing"]), deadline.DEFAULT_TIMEOUT_S) + + def test_an_explicit_timeout_still_wins(self) -> None: + self.assertEqual(self._timeout(["staleness", "--timeout", "60"]), 60.0) + + def test_the_help_text_names_the_default_actually_in_force(self) -> None: + # The help was a static string built in the shared helper, so the + # `set_defaults` override changed the value and left `--help` saying + # 900 -- three lines under a description announcing that staleness + # defaults to twice the usual. One screen contradicting itself. + import contextlib + import io + + def help_for(command): + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + with self.assertRaises(SystemExit): + cli.build_parser().parse_args([command, "--help"]) + return " ".join(buffer.getvalue().split()) + + self.assertIn("Default: 1800 (30 minutes)", help_for("staleness")) + self.assertIn("Default: 900 (15 minutes)", help_for("behavioral")) + + def test_the_help_text_lists_staleness_as_a_command_sharing_the_bound(self) -> None: + import contextlib + import io + + buffer = io.StringIO() + with contextlib.redirect_stdout(buffer): + with self.assertRaises(SystemExit): + cli.build_parser().parse_args(["staleness", "--help"]) + self.assertIn("behavioral and staleness", " ".join(buffer.getvalue().split())) + + + if __name__ == "__main__": unittest.main(verbosity=2)