From 25b904feef365f9e9eda71f21f9360b7bb2a1e9e Mon Sep 17 00:00:00 2001 From: John Lybeck Date: Tue, 17 Nov 2026 14:55:29 +0100 Subject: [PATCH] Let a repo set the dataset bar its skills are held to Tier 0 was three positive evaluations and two negative, as constants. That is the right bar for a catalogue that starts with this harness and the wrong one for a catalogue that predates it: fifty skills written before these checks existed cannot produce two hundred and fifty prompts before their first green run, so the choice on offer was to run none of the checks at all. The folder, frontmatter and reference checks are worth having on day one, and they were being held hostage by the dataset requirement. --min-positive and --min-negative lower the bar and raise it again as the datasets land. Zero for both stops requiring a dataset; a skill that ships one anyway is still read against whatever the bar is, because "not required of every skill" is not "unread where present". Where that bar may be set is the point, so it is a flag and nothing else. No file in the repo under test, none inside a skill, and no environment variable: the bar a skill is measured against is not the skill author's to set, and a file beside the skill would let one pull request add a thin dataset and lower the number judging it, leaving a reviewer to notice the second change to understand the first. On the command line it lives in the workflow, which is a path a repo can put behind branch protection and code owners. Two tests assert that property rather than trusting it. The applied bar is printed at the top of every structural run, passing or failing, so a report says what it was holding skills to. Signed-off-by: John Lybeck --- .github/workflows/reusable.yml | 27 ++++++ .github/workflows/skill-evals.yml | 36 +++++++- docs/usage.md | 25 ++++++ skillscope/cli.py | 60 ++++++++++++- skillscope/config.py | 56 ++++++++++++ skillscope/datasets.py | 43 +++++---- tests/test_skillscope.py | 140 +++++++++++++++++++++++++++++- 7 files changed, 367 insertions(+), 20 deletions(-) diff --git a/.github/workflows/reusable.yml b/.github/workflows/reusable.yml index 7db00d8..42cfcab 100644 --- a/.github/workflows/reusable.yml +++ b/.github/workflows/reusable.yml @@ -107,6 +107,22 @@ on: description: "`runs-on` for every job: one label, or a JSON array of them." type: string default: ubuntu-latest + min_positive: + description: >- + Evaluations a skill needs with `skill_should_trigger: true`. Blank + uses skillscope's own default. Zero for both this and min_negative + means a skill need not ship a dataset at all, which is how a + catalogue adopts these checks before it has written one. It lives + here, in the workflow, because the bar a skill is measured against is + not the skill author's to set. + type: string + default: "" + min_negative: + description: >- + Evaluations a skill needs with `skill_should_trigger: false`. Blank + uses skillscope's own default. See min_positive. + type: string + default: "" min_accuracy: description: >- Fail a routing leg below this accuracy (0-1). The default fails on any @@ -285,6 +301,13 @@ jobs: uses: ./.skillscope-action with: command: structural + # `!= ''` rather than the bare truthiness the inputs above use: + # `0` is a meaningful value for these two, and the condition + # should say "the caller filled it in" rather than rely on a + # cast that happens to agree. + args: >- + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ needs.discover.outputs.globs }} routing: @@ -337,6 +360,8 @@ jobs: args: >- --routing-room '${{ matrix.skill }}' --min-accuracy '${{ inputs.min_accuracy }}' + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} --output routing-report.json --keep-logs routing-logs skills: ${{ needs.discover.outputs.globs }} @@ -394,6 +419,8 @@ jobs: args: >- --skill '${{ matrix.skill }}' --output behavioral-report.json + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ needs.discover.outputs.globs }} install-claude: "true" diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index 78c2a5f..78666a3 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -111,6 +111,22 @@ on: that answer a runner's IP with a 403. type: string default: "" + min_positive: + description: >- + Evaluations a skill needs with `skill_should_trigger: true`. Blank + uses skillscope's own default. Zero for both this and min_negative + means a skill need not ship a dataset at all, which is how a + catalogue adopts these checks before it has written one. It lives + here, in the workflow, because the bar a skill is measured against is + not the skill author's to set. + type: string + default: "" + min_negative: + description: >- + Evaluations a skill needs with `skill_should_trigger: false`. Blank + uses skillscope's own default. See min_positive. + type: string + default: "" skill_files: description: >- Files every skill must ship beside its SKILL.md, comma-separated and @@ -349,10 +365,16 @@ jobs: uses: ./.skillscope-action with: command: structural + # `!= ''` rather than the bare truthiness the inputs above use: + # `0` is a meaningful value for these two, and the condition + # should say "the caller filled it in" rather than rely on a + # cast that happens to agree. args: >- ${{ inputs.doc_globs && format('--docs ''{0}''', inputs.doc_globs) || '' }} ${{ inputs.skill_files && format('--skill-files ''{0}''', inputs.skill_files) || '' }} ${{ inputs.skill_sections && format('--skill-sections ''{0}''', inputs.skill_sections) || '' }} + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ inputs.skill_globs }} python-version: ${{ inputs.python_version }} @@ -504,6 +526,8 @@ jobs: --external ${{ inputs.doc_globs && format('--docs ''{0}''', inputs.doc_globs) || '' }} ${{ inputs.excluded_urls && format('--exclude-url ''{0}''', inputs.excluded_urls) || '' }} + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ inputs.skill_globs }} python-version: ${{ inputs.python_version }} @@ -565,6 +589,8 @@ jobs: ${{ inputs.only && format('--only ''{0}''', inputs.only) || '' }} --output routing-report.json --keep-logs routing-logs + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ inputs.skill_globs }} python-version: ${{ inputs.python_version }} setup-python: "false" @@ -627,7 +653,10 @@ jobs: uses: ./.skillscope-action with: command: behavioral - args: --skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }} + args: >- + --skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }} + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ inputs.skill_globs }} python-version: ${{ inputs.python_version }} setup-python: "false" @@ -684,7 +713,10 @@ jobs: uses: ./.skillscope-action with: command: behavioral - args: --skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }} + args: >- + --skill '${{ matrix.skill }}' ${{ needs.discover.outputs.extended }} + ${{ inputs.min_positive != '' && format('--min-positive ''{0}''', inputs.min_positive) || '' }} + ${{ inputs.min_negative != '' && format('--min-negative ''{0}''', inputs.min_negative) || '' }} skills: ${{ inputs.skill_globs }} python-version: ${{ inputs.python_version }} setup-python: "false" diff --git a/docs/usage.md b/docs/usage.md index ee4c527..7a3468f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -29,6 +29,27 @@ Every skill ships one dataset at `/evals/evals.json`. A prompt alone grades routing; add expectations and it also runs end to end. How to write one is in [authoring-evals.md](authoring-evals.md). +By default a skill needs three evaluations that expect it to fire and two that +expect it not to, and a skill with no dataset at all fails the structural +check. That is the bar a new catalogue should hold, and it is the wrong bar for +an existing one: a repo adopting skillscope over fifty skills written before it +cannot write two hundred and fifty prompts before its first green run, and the +alternative it reaches for is to not run the checks. `--min-positive` and +`--min-negative` lower the bar so the folder, frontmatter and reference checks +can start working immediately, and get raised as the datasets land. Zero for +both turns the dataset requirement off; a skill that ships a dataset anyway is +still read against whatever the bar is. + +**Where that bar may be set is the point.** It is a flag, so under CI it is a +line in the workflow, which is a path a repo can put behind branch protection +and code owners. There is no file in the repo under test, none inside a skill, +and no environment variable the CLI will read. The bar a skill is measured +against is not the skill author's to set: were it a file beside the skill, the +same pull request that adds a thin dataset could lower the number it is judged +by, and a reviewer would have to notice the second change to understand the +first. The applied bar is printed at the top of every structural run, passing +or failing, so a report always says what it was holding skills to. + ## Configuring the repo under test There is no config file. Everything about your repo is a CLI flag, or the same @@ -46,6 +67,8 @@ places to disagree. | `--exclude-url` | `excluded_urls` | none | Regexes matching URLs the external reference check leaves alone. For hosts that are auth-gated or that answer a runner's IP with a 403. | | `--skill-files` | `skill_files` | none | Files every skill must ship beside its `SKILL.md`, such as a governance card. | | `--skill-sections` | `skill_sections` | none | `##` headings each of those markdown files must have something under, such as `Description,Owner,License`. | +| `--min-positive` | `min_positive` | `3` | Evaluations a skill needs with `skill_should_trigger: true`. | +| `--min-negative` | `min_negative` | `2` | Evaluations it needs with `skill_should_trigger: false`. Zero for both means a skill need not ship a dataset at all. | | `--behavior-runner` | `behavior_runner` | `["ubuntu-latest"]` | `runs-on` labels for a behavioral leg. | | `--behavior-os` | `behavior_os` | `Linux` | Platforms a skill runs on when its `machine.yml` does not narrow them. | | `--scoped-runner` | `scoped_runner` | reuses `behavior_runner` | Base labels for a leg whose skill asks for hardware labels. | @@ -410,6 +433,8 @@ Naming several, and holding them to different bars: | `behavioral` | `required` | `required`, `optional`, or `off`. | | `runner` | `ubuntu-latest` | `runs-on` for every job: one label, or a JSON array of them. | | `min_accuracy` | `1` | The routing bar. `0` reports the score without gating on it. | +| `min_positive` | (skillscope's own) | Evaluations a skill needs with `skill_should_trigger: true`. | +| `min_negative` | (skillscope's own) | Evaluations it needs with `skill_should_trigger: false`. Zero for both means a skill need not ship a dataset. | | `api_key` | (none) | The model API key, mapped from the caller's vault. One secret, not the whole set. | | `api_key_secret` | `ANTHROPIC_API_KEY` | Name to look up under `secrets: inherit`, if you would rather pass the vault than map one key. | | `api_base_url` | (none) | Base URL of the gateway that issued the key. Blank talks to the provider directly. | diff --git a/skillscope/cli.py b/skillscope/cli.py index bc9c406..74cb18a 100644 --- a/skillscope/cli.py +++ b/skillscope/cli.py @@ -299,12 +299,32 @@ def routing_gate(totals: dict, min_accuracy: float) -> str | None: def cmd_structural(args: argparse.Namespace) -> int: """Structural checks over every skill folder, dataset, and reference.""" + # Before the gate, not after it: a run that fails still has to say which + # bar it was holding skills to. Printing this only on success would hide + # exactly the case worth seeing -- a repo whose datasets are not required, + # failing on something else -- behind a passing run. + cfg = config.active() + # Flushed, because the failures below go to stderr: unflushed stdout is + # block-buffered down a pipe, and the line would land after the report it + # is meant to frame in every CI log that captures both. + if cfg.min_positive or cfg.min_negative: + print( + f"[evals] dataset bar: {cfg.min_positive} positive, " + f"{cfg.min_negative} negative per skill.", + flush=True, + ) + else: + print( + "[evals] dataset bar: off. Skills are not required to ship " + "`evals/evals.json`; the ones that do are still read.", + flush=True, + ) + found = _structural_or_exit() skills = datasets.skills_with_datasets() # Extended datasets are checked regardless of --extended, so count them # here too rather than reporting fewer cases than were checked. cases = datasets.load_all_cases(extended=True) - cfg = config.active() print(f"[evals] OK: {len(cfg.skills)} skill folder(s) in shape.") print( f"[evals] OK: {len(cases)} case(s) across {len(skills)} skill(s) " @@ -882,6 +902,37 @@ def _add_skills_argument(parser: argparse.ArgumentParser) -> None: ) +def _add_dataset_bar_arguments(parser: argparse.ArgumentParser) -> None: + """How much of a dataset this repo asks of a skill. + + A flag and nothing else. The bar a skill is measured against is the repo's + policy, not the skill author's, so it has to live somewhere a skill cannot + reach: see `min_positive` on `config.Config`. + """ + parser.add_argument( + "--min-positive", + default="", + metavar="N", + help=( + "Evaluations a skill needs with `skill_should_trigger: true`. " + f"Default: {config.DEFAULT_MIN_POSITIVE}. Lower it to adopt these " + "checks over a catalogue that predates them, then raise it as the " + "datasets land." + ), + ) + parser.add_argument( + "--min-negative", + default="", + metavar="N", + help=( + "Evaluations a skill needs with `skill_should_trigger: false`. " + f"Default: {config.DEFAULT_MIN_NEGATIVE}. Zero for both this and " + "--min-positive means a skill need not ship a dataset at all; the " + "folder, frontmatter and reference checks still run." + ), + ) + + def _add_docs_argument(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--docs", @@ -924,6 +975,10 @@ def _add_routing_room_argument( def _add_graded_arguments(parser: argparse.ArgumentParser) -> None: """Flags shared by the routing and behavioral commands.""" _add_skills_argument(parser) + # A graded run gates on the same dataset bar before it spends anything, so + # it takes the same flags. Without them a workflow that lowered the bar for + # `structural` would still be refused by `routing` on the next line. + _add_dataset_bar_arguments(parser) parser.add_argument( "--skill", default="", @@ -1114,6 +1169,7 @@ def build_parser() -> argparse.ArgumentParser: ), ) _add_skills_argument(structural_parser) + _add_dataset_bar_arguments(structural_parser) _add_docs_argument(structural_parser) structural_parser.add_argument( "--skill-files", @@ -1341,6 +1397,8 @@ def _configure(args: argparse.Namespace) -> None: "behavior_runner", "behavior_os", "scoped_runner", + "min_positive", + "min_negative", ) } settings["scoped_gate"] = getattr(args, "scoped_gate", "") or "" diff --git a/skillscope/config.py b/skillscope/config.py index bc7d98a..62e7aa9 100644 --- a/skillscope/config.py +++ b/skillscope/config.py @@ -61,6 +61,15 @@ DEFAULT_BEHAVIOR_RUNNER = ("ubuntu-latest",) DEFAULT_BEHAVIOR_OS = ("Linux",) +# Tier 0, the bar every skill clears before it can ship: five prompts, no +# hardware, no assertions, enforced structurally so a thin dataset fails +# without spending a token. These are the defaults, not the rule -- a repo +# adopting this harness over an existing catalogue lowers them, and raises them +# again as the datasets land. Where that decision may be made is the point of +# `min_positive` on Config below. +DEFAULT_MIN_POSITIVE = 3 +DEFAULT_MIN_NEGATIVE = 2 + # The two answers `--routing-room` takes instead of a list of names: every # skill in the repo that ships a dataset, and no routing run at all. Both are # words rather than shapes of an empty flag, because saying nothing means "work @@ -123,6 +132,24 @@ class Config: scoped_gate: str = "" scoped_environment: str = "" + # How many of a skill's evaluations must expect it to fire, and how many + # must expect it not to. Zero for both means a skill need not ship a + # dataset at all, which is how a catalogue adopts the structural checks + # before it has written one. + # + # This is reachable from the command line and from nowhere else, and that + # is the design rather than an omission. The bar a skill is held to is not + # the skill author's to set: a file inside a skill, or at the repo root, + # would let the same pull request that adds a thin dataset also lower the + # number it is measured against, and the reviewer would have to notice the + # second change to understand the first. On the command line it lives in + # the workflow, which is a path a repo can put behind branch protection and + # code owners. There is deliberately no environment variable either: a + # self-hosted runner's environment is set by whoever administers the + # machine, which is not the same set of people who review a workflow diff. + min_positive: int = DEFAULT_MIN_POSITIVE + min_negative: int = DEFAULT_MIN_NEGATIVE + @property def skills(self) -> dict[str, Path]: """Every skill this repo declares, as ``{name: folder}``. @@ -286,6 +313,31 @@ def wants_no_skills(value: object) -> bool: return _is_sentinel(value, NO_SKILLS) +def _bar(value: object, flag: str, default: int) -> int: + """A non-negative count from a flag, a workflow input, or a test fixture. + + Blank means the default, so a workflow input nobody filled in is the same + as no flag at all. Anything else has to be a whole number that is not + negative: a bar of -1 would read as "off" to whoever wrote it and as "every + count clears it" to the code, and those are the same outcome reached by + accident. Zero is the way to say off, and it says it once. + """ + if value is None or (isinstance(value, str) and not value.strip()): + return default + try: + count = int(str(value).strip()) + except ValueError: + raise SystemExit( + f"error: {flag} takes a whole number, not {value!r}." + ) from None + if count < 0: + raise SystemExit( + f"error: {flag} cannot be negative (got {count}). Zero is how a " + "repo says this bar does not apply yet." + ) + return count + + def build( root: Path | None = None, *, @@ -301,6 +353,8 @@ def build( scoped_runner: object = None, scoped_gate: str = "", scoped_environment: str = "", + min_positive: object = None, + min_negative: object = None, dataset_skills: list[str] | None = None, ) -> Config: """A Config from loose values: what the CLI hands over after parsing. @@ -347,6 +401,8 @@ def build( scoped_runner=_items(scoped_runner, "--scoped-runner"), scoped_gate=(scoped_gate or "").strip(), scoped_environment=(scoped_environment or "").strip(), + min_positive=_bar(min_positive, "--min-positive", DEFAULT_MIN_POSITIVE), + min_negative=_bar(min_negative, "--min-negative", DEFAULT_MIN_NEGATIVE), ) diff --git a/skillscope/datasets.py b/skillscope/datasets.py index aea25f9..82acf65 100644 --- a/skillscope/datasets.py +++ b/skillscope/datasets.py @@ -91,11 +91,13 @@ # What a new skill copies to start. TEMPLATE = PACKAGE_DIR / "data" / "TEMPLATE.json" -# Tier 0, the bar every skill clears before it can ship. Cheap to meet (five -# prompts, no hardware, no assertions) and enforced structurally so a thin -# dataset fails the structural checks without spending a single token. -MIN_POSITIVE_CASES = 3 -MIN_NEGATIVE_CASES = 2 +# Tier 0, the bar every skill clears before it can ship. Cheap to meet by +# default (five prompts, no hardware, no assertions) and enforced structurally +# so a thin dataset fails without spending a single token. The numbers live on +# the config rather than here, because a repo adopting this harness over a +# catalogue that predates it needs to lower them and ratchet them back up; see +# `min_positive` on `config.Config` for why the command line is the only place +# that decision can be made. # A dataset is one array of evaluations, and every evaluation answers the # routing question outright rather than leaving it to be inferred. @@ -504,33 +506,44 @@ def structural_errors(skills: list[str] | None = None) -> list[str]: def tier0_errors(skill: str, cases: list[Case]) -> list[str]: - """Whether `skill` meets the mandatory coverage bar. + """Whether `skill` meets this repo's coverage bar. - Counted over the required dataset alone (evals.json). + Counted over the required dataset alone (evals.json). The bar is + `min_positive` and `min_negative` on the active config, which only the + command line sets; both at zero is a repo saying datasets are not required + of it yet, and then a skill without one is not a finding. """ + cfg = config.active() + required = cfg.min_positive or cfg.min_negative + if not dataset_path(skill).is_file(): + if not required: + return [] return [ f"{skill}: no eval dataset. Every skill needs " f"`{DATASET_RELPATH.as_posix()}` in its folder with at least " - f"{MIN_POSITIVE_CASES} evaluations where `{TRIGGER_KEY}` is true " - f"and {MIN_NEGATIVE_CASES} where it is false. " + f"{cfg.min_positive} evaluations where `{TRIGGER_KEY}` is true " + f"and {cfg.min_negative} where it is false. " "Copy the template from `skillscope template` to start." ] + # A dataset that exists is still read against the bar, whatever the bar is. + # A repo that has turned the requirement off has said it does not yet + # expect every skill to ship one, not that the ones that do may be empty. errors: list[str] = [] positive = sum(1 for c in cases if c.skill_should_trigger) negative = len(cases) - positive - if positive < MIN_POSITIVE_CASES: + if positive < cfg.min_positive: errors.append( f"{skill}: {positive} evaluation(s) with `{TRIGGER_KEY}: true`; " - f"Tier 0 needs at least {MIN_POSITIVE_CASES}. Add prompts a real " - "user would type." + f"this repo asks for at least {cfg.min_positive}. Add prompts a " + "real user would type." ) - if negative < MIN_NEGATIVE_CASES: + if negative < cfg.min_negative: errors.append( f"{skill}: {negative} evaluation(s) with `{TRIGGER_KEY}: false`; " - f"Tier 0 needs at least {MIN_NEGATIVE_CASES}. Add prompts close to " - "this skill's domain that should NOT trigger it." + f"this repo asks for at least {cfg.min_negative}. Add prompts " + "close to this skill's domain that should NOT trigger it." ) return errors diff --git a/tests/test_skillscope.py b/tests/test_skillscope.py index 8fb988d..62dbcec 100644 --- a/tests/test_skillscope.py +++ b/tests/test_skillscope.py @@ -104,11 +104,11 @@ def tier0_dataset(slug: str, **extra) -> dict: """The smallest dataset that clears the mandatory coverage bar.""" evaluations = [ {"id": f"{slug}-yes-{i}", TRIGGER_KEY: True, "prompt": f"please do {slug} work {i}"} - for i in range(datasets.MIN_POSITIVE_CASES) + for i in range(config.DEFAULT_MIN_POSITIVE) ] evaluations += [ {"id": f"{slug}-no-{i}", TRIGGER_KEY: False, "prompt": f"nothing to do with {slug} {i}"} - for i in range(datasets.MIN_NEGATIVE_CASES) + for i in range(config.DEFAULT_MIN_NEGATIVE) ] return {EVALUATIONS_KEY: evaluations, **extra} @@ -1393,6 +1393,142 @@ def test_a_skill_with_no_dataset_is_reported(self) -> None: self.assertIn("no eval dataset", errors[0]) +class TestTheDatasetBarIsTheRepos(unittest.TestCase): + """The coverage bar is configuration, and only the command line sets it. + + A repo adopting these checks over a catalogue written before them has to be + able to lower the bar, or it runs none of the checks at all. What that must + never become is a number the skill under review can move: the point of the + bar is that something outside the skill decides it. + """ + + def setUp(self) -> None: + self.repo = Repo(self) + + def test_a_lowered_bar_accepts_what_the_default_rejects(self) -> None: + self.repo.skill("demo-skill", dataset=tier0_dataset("demo")) + self.repo.activate() + thin, _ = parse( + { + EVALUATIONS_KEY: [ + {"id": "yes", TRIGGER_KEY: True, "prompt": "do demo work"}, + {"id": "no", TRIGGER_KEY: False, "prompt": "unrelated"}, + ] + }, + skill="demo-skill", + ) + self.assertNotEqual(datasets.tier0_errors("demo-skill", thin), []) + self.repo.reactivate(min_positive=1, min_negative=1) + self.assertEqual(datasets.tier0_errors("demo-skill", thin), []) + + def test_a_raised_bar_rejects_what_the_default_accepts(self) -> None: + self.repo.skill("demo-skill", dataset=tier0_dataset("demo")) + self.repo.activate(min_positive=99) + cases, _ = parse(tier0_dataset("demo"), skill="demo-skill") + errors = datasets.tier0_errors("demo-skill", cases) + self.assertTrue(any("at least 99" in e for e in errors), errors) + + def test_zero_for_both_stops_requiring_a_dataset(self) -> None: + self.repo.skill("bare-skill") + self.repo.activate() + self.assertNotEqual(datasets.tier0_errors("bare-skill", []), []) + self.repo.reactivate(min_positive=0, min_negative=0) + self.assertEqual(datasets.tier0_errors("bare-skill", []), []) + + def test_one_bar_still_standing_still_requires_a_dataset(self) -> None: + """Only *both* at zero means "not required"; one of them does not.""" + self.repo.skill("bare-skill") + self.repo.activate(min_positive=0, min_negative=1) + errors = datasets.tier0_errors("bare-skill", []) + self.assertEqual(len(errors), 1) + self.assertIn("no eval dataset", errors[0]) + + def test_a_dataset_that_exists_is_read_even_with_the_bar_off(self) -> None: + """Off means "not required of every skill", not "unread where present".""" + self.repo.skill("demo-skill", dataset=tier0_dataset("demo")) + self.repo.activate(min_positive=0, min_negative=0) + self.assertIn("demo-skill", datasets.skills_with_datasets()) + cases = datasets.routing_cases(["demo-skill"]) + self.assertTrue(cases) + + def test_the_bar_is_not_settable_from_inside_a_skill(self) -> None: + """The governance property, asserted rather than assumed. + + A file a skill author can edit must not move the number their own + skill is judged by, whatever it is called or wherever in the skill it + sits. Nothing reads these; the test exists so that stays true. + """ + folder = self.repo.skill("bare-skill") + for name in ("skillscope.json", "config.yml", "evals/skillscope.yml"): + path = folder / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + '{"min_positive": 0, "min_negative": 0}', encoding="utf-8" + ) + self.repo.activate() + errors = datasets.tier0_errors("bare-skill", []) + self.assertEqual(len(errors), 1) + self.assertIn("no eval dataset", errors[0]) + + def test_the_bar_is_not_settable_from_the_environment(self) -> None: + """No environment fallback, unlike SKILLSCOPE_SKILLS. + + A self-hosted runner's environment is set by whoever administers the + machine, who is not the person reviewing the workflow diff. Adding a + fallback here would quietly widen who can lower the bar. + """ + self.repo.skill("bare-skill") + for key in ("SKILLSCOPE_MIN_POSITIVE", "SKILLSCOPE_MIN_NEGATIVE"): + os.environ[key] = "0" + self.addCleanup(os.environ.pop, key, None) + self.repo.activate() + errors = datasets.tier0_errors("bare-skill", []) + self.assertEqual(len(errors), 1) + self.assertIn("no eval dataset", errors[0]) + + def test_a_bar_that_is_not_a_count_is_refused(self) -> None: + for value in ("-1", "two", "1.5"): + with self.subTest(value=value): + with self.assertRaises(SystemExit) as caught: + config.build(self.repo.root, min_positive=value) + self.assertIn("--min-positive", str(caught.exception)) + + def test_a_blank_bar_is_the_default(self) -> None: + """A workflow input nobody filled in is the same as no flag at all.""" + cfg = config.build(self.repo.root, min_positive="", min_negative=None) + self.assertEqual(cfg.min_positive, config.DEFAULT_MIN_POSITIVE) + self.assertEqual(cfg.min_negative, config.DEFAULT_MIN_NEGATIVE) + + def _structural_stdout(self, *flags: str) -> str: + """Run `structural` the way CI does, through argparse, and read stdout.""" + previous = config.use(None) + self.addCleanup(config.use, previous) + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + code = cli.main(["--repo", str(self.repo.root), "structural", *flags]) + self.assertEqual(code, 0, stdout.getvalue()) + return stdout.getvalue() + + def test_the_applied_bar_is_printed_whether_it_is_on_or_off(self) -> None: + """The one test here that goes in at the flags rather than the config. + + Every other test in this class calls `config.build` directly, so all + of them would keep passing if the flags stopped reaching it. This one + takes the path CI takes, and reads back the line that tells a reviewer + which bar a given run was holding its skills to. + """ + self.repo.skill("demo-skill", dataset=tier0_dataset("demo")) + self.assertIn( + f"dataset bar: {config.DEFAULT_MIN_POSITIVE} positive, " + f"{config.DEFAULT_MIN_NEGATIVE} negative", + self._structural_stdout(), + ) + self.assertIn( + "dataset bar: off", + self._structural_stdout("--min-positive", "0", "--min-negative", "0"), + ) + + class TestExtendedDataset(unittest.TestCase): """The optional second dataset: same format, no bar, opt-in run."""