From 9413926c9362eb499b350a9baa7e6a13ee7a0d6e Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:42:00 +0100 Subject: [PATCH 1/3] fix: restore the review gate for external APM primitives - Add Label Sync (labels-base.yaml from DevSecNinja/.github plus repo labels.yaml) so apm-sync, external-primitives and merge: manual exist; gh pr create failed on the missing apm-sync label on every materialize run. - Renovate: github/awesome-copilot bumps never auto-merge and get external-primitives + merge: manual. - APM Materialize labels its PR for manual review and re-applies apm-overrides.json after apm install, keeping the Sonnet 5 pin from #135. - Document the review step in docs/apm.md and CONTRIBUTING. Refs #210 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 3 + .github/labels-base.yaml | 63 +++++++++++ .github/labels.yaml | 36 +++++++ .github/workflows/apm-materialize.yml | 20 +++- .github/workflows/label-sync.yml | 31 ++++++ .github/workflows/markdown-validation.yml | 9 ++ .lefthook.toml | 4 + CONTRIBUTING.md | 12 +++ apm-overrides.json | 12 +++ docs/apm.md | 50 +++++++++ renovate.json5 | 14 +++ scripts/apply-apm-overrides.py | 122 ++++++++++++++++++++++ tests/test_apm_overrides.py | 100 ++++++++++++++++++ 13 files changed, 474 insertions(+), 2 deletions(-) create mode 100644 .github/labels-base.yaml create mode 100644 .github/labels.yaml create mode 100644 .github/workflows/label-sync.yml create mode 100644 apm-overrides.json create mode 100755 scripts/apply-apm-overrides.py create mode 100644 tests/test_apm_overrides.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 28ef0ec..7f05f18 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -154,6 +154,9 @@ helper, run `PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 3. **`description` and `category` are required** in every primitive's frontmatter. 4. **Bash scripts are executable** — run as `bash script-name.sh`. 5. **No background processes** — all commands complete immediately. +6. **Never hand-edit vendored agents** in `.github/agents/` that come from + `apm.yml` dependencies; `apm install` reverts them. Put local pins in + `apm-overrides.json` (see the review gate section of `docs/apm.md`). ## Coding Style Preferences diff --git a/.github/labels-base.yaml b/.github/labels-base.yaml new file mode 100644 index 0000000..c3b1e55 --- /dev/null +++ b/.github/labels-base.yaml @@ -0,0 +1,63 @@ +--- +# Base labels shared across all DevSecNinja repositories. +# Repo-specific labels (e.g. area/* labels) should be defined in each repo's +# .github/labels.yaml file. + +# Renovate Types +- name: renovate/container + color: "027fa0" +- name: renovate/github-action + color: "027fa0" +- name: renovate/github-release + color: "027fa0" +- name: renovate/github-tag + color: "027fa0" +- name: renovate/gomod + color: "027fa0" +- name: renovate/mise + color: "027fa0" +- name: renovate/ansible + color: "027fa0" + +# Semantic Types +- name: type/digest + color: "ffeC19" +- name: type/patch + color: "ffeC19" +- name: type/minor + color: "ff9800" +- name: type/major + color: "f6412d" + +# Automation +- name: "merge: auto" + color: "0075ca" + description: PR is eligible for automerge + aliases: + - automerge +- name: "merge: manual" + color: "5319e7" + description: PR requires manual review and merge +- name: "autorelease: pending" + color: "ededed" + description: Release Please — release pending +- name: "autorelease: tagged" + color: "0e8a16" + description: Release Please — release tagged +- name: config-sync + color: "0075ca" +- name: apm-sync + color: "0075ca" + description: Automated APM materialize PR +- name: todo-to-issue + color: "0075ca" + +# Uncategorized +- name: bug + color: "d73a4a" +- name: enhancement + color: "a2eeef" +- name: hold + color: "ee0701" +- name: stale-dependency + color: "e4e669" diff --git a/.github/labels.yaml b/.github/labels.yaml new file mode 100644 index 0000000..83573a9 --- /dev/null +++ b/.github/labels.yaml @@ -0,0 +1,36 @@ +--- +# Labels specific to ai-toolkit. Label Sync merges this file with the org-wide +# .github/labels-base.yaml (synced from DevSecNinja/.github by config-sync) and +# deletes any label in neither file. + +# Review gate for third-party agent text (see docs/apm.md). Set on Renovate's +# github/awesome-copilot bumps and on the APM Materialize PR; neither may +# auto-merge. +- name: external-primitives + color: "b60205" + description: Changes third-party AI primitives; owner review required + +# GitHub's default labels, kept so the sync doesn't delete them. +- name: dependencies + color: "ededed" +- name: documentation + color: "0075ca" + description: Improvements or additions to documentation +- name: duplicate + color: "cfd3d7" + description: This issue or pull request already exists +- name: good first issue + color: "7057ff" + description: Good for newcomers +- name: help wanted + color: "008672" + description: Extra attention is needed +- name: invalid + color: "e4e669" + description: This doesn't seem right +- name: question + color: "d876e3" + description: Further information is requested +- name: wontfix + color: "ffffff" + description: This will not be worked on diff --git a/.github/workflows/apm-materialize.yml b/.github/workflows/apm-materialize.yml index 4a985e7..b3919b5 100644 --- a/.github/workflows/apm-materialize.yml +++ b/.github/workflows/apm-materialize.yml @@ -107,6 +107,12 @@ jobs: # are not needed to materialize the agents here). apm install --update --only apm --verbose + - name: Re-apply local overrides + # `apm install` rewrites the vendored agents, which drops local pins + # such as the security analyzer's model. apm-overrides.json keeps them. + if: steps.precheck.outputs.skip != 'true' + run: python3 scripts/apply-apm-overrides.py + - name: Fetch materialize branch # Populate refs/remotes/origin/chore/apm-materialize so that the # --force-with-lease push in open-pr has a valid lease to compare @@ -125,7 +131,13 @@ jobs: base: main title: "chore: materialize APM primitives" commit-message: "chore: materialize APM primitives from apm.yml" - labels: apm-sync + # Label Sync creates these from .github/labels*.yaml; `gh pr create` + # fails when one is missing. `external-primitives` marks the review + # gate and `merge: manual` that the owner merges this PR by hand. + labels: | + apm-sync + external-primitives + merge: manual github-token: ${{ steps.app-token.outputs.token }} # Stage the lockfile and deployed harness files (apm.yml itself is # already on main; apm_modules/ and the non-committed harness output @@ -138,6 +150,10 @@ jobs: `main` (e.g. a Renovate SHA/tag bump). **Review the `.github/agents/` diff** — this is the prompt-hijacking - gate: confirm the upstream agent text changed as expected. + gate: confirm the upstream agent text changed as expected. Local + pins from `apm-overrides.json` are re-applied on top. + + Never auto-merge this PR: the owner reviews and merges it. See + `docs/apm.md`. Triggered by `${{ github.workflow }}` at commit `${{ github.sha }}`. diff --git a/.github/workflows/label-sync.yml b/.github/workflows/label-sync.yml new file mode 100644 index 0000000..6e10835 --- /dev/null +++ b/.github/workflows/label-sync.yml @@ -0,0 +1,31 @@ +--- +name: Label Sync + +# Creates and updates this repo's labels from the org-wide base set +# (.github/labels-base.yaml, delivered by config-sync from DevSecNinja/.github) +# plus the repo-specific .github/labels.yaml. Automation depends on these +# labels: `gh pr create --label` fails when a label is missing, which is what +# kept APM Materialize from opening its review PR (#210). + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - .github/labels.yaml + - .github/labels-base.yaml + # Weekly drift correction for labels edited in the UI; changes to the label + # files themselves are applied by the push trigger above. + schedule: + - cron: "0 0 * * 1" + +permissions: + contents: read + +jobs: + label-sync: + uses: DevSecNinja/.github/.github/workflows/label-sync.yml@8ef60d65001c56a496d12f6ae0a7091c05db2e06 # v3.5.0 + permissions: + contents: read + issues: write diff --git a/.github/workflows/markdown-validation.yml b/.github/workflows/markdown-validation.yml index e1d6e7f..bb40858 100644 --- a/.github/workflows/markdown-validation.yml +++ b/.github/workflows/markdown-validation.yml @@ -11,7 +11,10 @@ on: - '.github/prompts/**/*.md' - 'tests/**' - 'scripts/generate-index.sh' + - 'scripts/apply-apm-overrides.py' - 'apm.yml' + - 'apm-overrides.json' + - '.github/agents/**' - 'apm.lock.yaml' - 'release-please-config.json' - '.release-please-manifest.json' @@ -23,7 +26,10 @@ on: - '.github/prompts/**/*.md' - 'tests/**' - 'scripts/generate-index.sh' + - 'scripts/apply-apm-overrides.py' - 'apm.yml' + - 'apm-overrides.json' + - '.github/agents/**' - 'apm.lock.yaml' - 'release-please-config.json' - '.release-please-manifest.json' @@ -70,6 +76,9 @@ jobs: - name: Check apm.yml does not depend on itself run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_manifest.py' -v + - name: Check vendored primitives keep their local overrides + run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_overrides.py' -v + consumer-install: name: Check Consumer Install runs-on: ubuntu-latest diff --git a/.lefthook.toml b/.lefthook.toml index d1f8aca..faaf18b 100644 --- a/.lefthook.toml +++ b/.lefthook.toml @@ -21,6 +21,10 @@ run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_ [pre-commit.commands.apm-manifest] run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_manifest.py'" +# Vendored agents must keep their apm-overrides.json pins (see #210). +[pre-commit.commands.apm-overrides] +run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_overrides.py'" + # Regenerate INDEX.md / README index from .apm/ and re-stage the result, so the # generated index never drifts from the primitives (the common contributor miss). [pre-commit.commands.generate-index] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07bc1e6..2a1b395 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -117,6 +117,8 @@ What runs on **pre-commit**: — checks release-please can still find and bump the version in `apm.yml`. - `PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_manifest.py'` — checks `apm.yml` doesn't list `ai-toolkit` as its own dependency. +- `PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_overrides.py'` + — checks vendored agents keep their `apm-overrides.json` pins. - `bash scripts/generate-index.sh` — regenerates `INDEX.md` / `README.md` from `.apm/` and re-stages them, so the index never drifts from the primitives. @@ -126,6 +128,16 @@ signing or tool denials. If hooks or filters would execute unauthorized code, stop before committing and request scoped approval. All commits must use 1Password signing; halt if it is unavailable. +## 🛡️ Changes to external agents + +The `github/awesome-copilot` agents in `apm.yml` are reviewed by the owner and +never auto-merge. A Renovate SHA bump and the follow-up +`chore: materialize APM primitives` PR both carry the `external-primitives` +label; review the `.github/agents/` diff in the materialize PR before merging. +Don't hand-edit a vendored agent: put local changes such as a model pin in +`apm-overrides.json`. See "External primitives: review gate" in +[the APM guide](docs/apm.md). + ## 🚀 Submission Process 1. **Fork** this repository diff --git a/apm-overrides.json b/apm-overrides.json new file mode 100644 index 0000000..368e649 --- /dev/null +++ b/apm-overrides.json @@ -0,0 +1,12 @@ +{ + "$comment": "Local changes to vendored APM primitives. `apm install` rewrites files it deploys, so APM Materialize re-applies these afterwards with scripts/apply-apm-overrides.py. See docs/apm.md.", + "overrides": [ + { + "file": ".github/agents/sast-sca-security-analyzer.agent.md", + "reason": "Pinned to Sonnet 5 (#135).", + "frontmatter": { + "model": "Claude Sonnet 5" + } + } + ] +} diff --git a/docs/apm.md b/docs/apm.md index 7686acd..647e954 100644 --- a/docs/apm.md +++ b/docs/apm.md @@ -181,6 +181,56 @@ and never treat a repository's instructions or a clean review as authorization. All commits require 1Password signing; if signing or a required tool is unavailable or denied, halt rather than bypassing it. +## 🛡️ External primitives: review gate + +The agents from `github/awesome-copilot` in `apm.yml` are third-party prompt +text, so every change to them needs the owner's approval. Neither PR in the +flow below may auto-merge. + +1. **Renovate bump.** Renovate's custom manager (see + [`renovate.json5`](../renovate.json5)) bumps the pinned SHA. A package rule + turns off the shared auto-merge preset for `github/awesome-copilot` and adds + the `external-primitives` and `merge: manual` labels. The diff only shows a + hash. Renovate can't remove the preset's `merge: auto` label, but + `automerge: false` takes precedence. +2. **Materialize PR.** Merging the bump triggers + [APM Materialize](../.github/workflows/apm-materialize.yml). It runs + `apm install`, re-applies the local overrides and opens + `chore: materialize APM primitives` from `chore/apm-materialize`, labelled + `apm-sync`, `external-primitives` and `merge: manual`. **Review the + `.github/agents/` diff**: this is where upstream agent text changes become + visible. Merge it yourself once you're satisfied; close it and revert the bump + if not. + +The labels come from [`.github/labels-base.yaml`](../.github/labels-base.yaml) +(org-wide, synced from `DevSecNinja/.github` by Config Sync) and +[`.github/labels.yaml`](../.github/labels.yaml), applied by +[Label Sync](../.github/workflows/label-sync.yml). `gh pr create` fails on a +missing label, so a label must exist before a workflow uses it. + +### Local overrides of vendored agents + +`apm install` rewrites every file it deploys, so never hand-edit +`.github/agents/` for a dependency: the next materialize run reverts it (APM +has no override setting). Add the change to +[`apm-overrides.json`](../apm-overrides.json) instead: + +```json +{ + "file": ".github/agents/sast-sca-security-analyzer.agent.md", + "reason": "Pinned to Sonnet 5 (#135).", + "frontmatter": { "model": "Claude Sonnet 5" } +} +``` + +Each entry sets top-level frontmatter keys to string values. +[`scripts/apply-apm-overrides.py`](../scripts/apply-apm-overrides.py) applies +them, and APM Materialize runs it after `apm install`. Run it yourself after a +local `apm install`. It fails when a file is missing, so drop stale entries +when an agent is removed. `tests/test_apm_overrides.py` fails if a committed +agent doesn't match its override. `apm audit` reports overridden files as +drift from the lockfile, which is expected. + ## 🏷️ Versioning Versioning is driven by [release-please](https://github.com/googleapis/release-please) diff --git a/renovate.json5 b/renovate.json5 index a40d7f9..c8ffba4 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -32,4 +32,18 @@ currentValueTemplate: "main", }, ], + packageRules: [ + { + // The shared autoMerge preset merges digest bumps unattended, but this + // bump PR only shows a SHA. The agent text it pulls in surfaces in the + // follow-up APM Materialize PR, so the owner reviews and merges both. + // Renovate adds labels from every matching rule and can't remove one, + // so the preset's `merge: auto` label stays; `automerge: false` wins. + description: "Never auto-merge github/awesome-copilot bumps: third-party agent text needs owner review", + matchDepNames: ["github/awesome-copilot"], + automerge: false, + commitMessageSuffix: "", + addLabels: ["external-primitives", "merge: manual"], + }, + ], } diff --git a/scripts/apply-apm-overrides.py b/scripts/apply-apm-overrides.py new file mode 100755 index 0000000..1ff7cd5 --- /dev/null +++ b/scripts/apply-apm-overrides.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Re-apply local overrides to vendored APM primitives. + +`apm install` rewrites every file it deploys, so a hand edit to a vendored +agent (for example a model pin) is lost on the next APM Materialize run. Keep +such edits in apm-overrides.json instead; APM Materialize runs this script +right after `apm install`. Each override sets top-level frontmatter keys of +one deployed file to string values. Standard library only. + +Usage: apply-apm-overrides.py [--check] [--root DIR] + + --check list files that don't match their overrides and exit 1; write nothing. +""" + +import argparse +import json +from pathlib import Path, PurePosixPath +import re +import sys + + +CONFIG = "apm-overrides.json" +KEY = re.compile(r"[A-Za-z0-9_-]+") + + +class OverrideError(Exception): + pass + + +def load_overrides(root): + try: + data = json.loads((root / CONFIG).read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise OverrideError(f"{CONFIG}: {error}") from error + overrides = data.get("overrides") if isinstance(data, dict) else None + if not isinstance(overrides, list): + raise OverrideError(f"{CONFIG}: `overrides` must be a list") + for number, override in enumerate(overrides, 1): + where = f"{CONFIG}: override {number}" + if not isinstance(override, dict): + raise OverrideError(f"{where} must be an object") + path, values = override.get("file"), override.get("frontmatter") + if not isinstance(path, str) or not path: + raise OverrideError(f"{where} needs a `file`") + parts = PurePosixPath(path) + if parts.is_absolute() or ".." in parts.parts or "\\" in path: + raise OverrideError(f"{where}: `file` must be a relative path inside the repo") + if not isinstance(values, dict) or not values: + raise OverrideError(f"{where} needs a non-empty `frontmatter` object") + for key, value in values.items(): + if not KEY.fullmatch(key) or not isinstance(value, str): + raise OverrideError(f"{where}: frontmatter `{key}` must be a plain key with a string value") + return overrides + + +def set_frontmatter(text, values): + """Return `text` with top-level frontmatter `values` set; add missing keys.""" + lines = text.splitlines(keepends=True) + if not lines or lines[0].rstrip("\r\n") != "---": + raise OverrideError("no YAML frontmatter") + end = next((i for i in range(1, len(lines)) if lines[i].rstrip("\r\n") == "---"), None) + if end is None: + raise OverrideError("unterminated YAML frontmatter") + newline = "\r\n" if lines[0].endswith("\r\n") else "\n" + for key, value in values.items(): + line = f"{key}: {json.dumps(value, ensure_ascii=False)}{newline}" + pattern = re.compile(rf"{re.escape(key)}[ \t]*:") + matches = [i for i in range(1, end) if pattern.match(lines[i])] + if len(matches) > 1: + raise OverrideError(f"frontmatter key `{key}` appears more than once") + if not matches: + lines.insert(end, line) + end += 1 + continue + start = stop = matches[0] + # Swallow a multi-line value: indented lines or a column-0 `- ` list. + while stop + 1 < end and re.match(r"[ \t]|-([ \t]|$)", lines[stop + 1]): + stop += 1 + lines[start:stop + 1] = [line] + end -= stop - start + return "".join(lines) + + +def apply(root, check=False): + """Apply every override under `root`; return the files that changed.""" + changed = [] + for override in load_overrides(root): + path = root / override["file"] + try: + text = path.read_bytes().decode("utf-8") + except OSError as error: + raise OverrideError( + f"{override['file']}: {error.strerror}; drop or update its entry in {CONFIG}" + ) from error + try: + updated = set_frontmatter(text, override["frontmatter"]) + except OverrideError as error: + raise OverrideError(f"{override['file']}: {error}") from error + if updated != text: + changed.append(override["file"]) + if not check: + path.write_bytes(updated.encode("utf-8")) + return changed + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--check", action="store_true", help="report drift without writing") + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + args = parser.parse_args(argv) + try: + changed = apply(args.root, check=args.check) + except OverrideError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + for path in changed: + print(f"{'needs override' if args.check else 'applied override'}: {path}") + return 1 if args.check and changed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_apm_overrides.py b/tests/test_apm_overrides.py new file mode 100644 index 0000000..e7046d1 --- /dev/null +++ b/tests/test_apm_overrides.py @@ -0,0 +1,100 @@ +"""Tests for scripts/apply-apm-overrides.py (see #210). + +`apm install` rewrites vendored agents, so local pins live in +apm-overrides.json and APM Materialize re-applies them. Standard library only. +""" + +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("apply_apm_overrides", ROOT / "scripts" / "apply-apm-overrides.py") +overrides = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(overrides) + +AGENT = ( + "---\n" + 'description: "Scans code"\n' + 'model: "Claude Sonnet 4.6"\n' + 'tools: ["search"]\n' + "---\n" + "\n" + "model: body text is not frontmatter\n" +) + + +class SetFrontmatterTests(unittest.TestCase): + def test_replaces_existing_key_and_leaves_the_rest(self): + result = overrides.set_frontmatter(AGENT, {"model": "Claude Sonnet 5"}) + self.assertEqual(result, AGENT.replace('"Claude Sonnet 4.6"', '"Claude Sonnet 5"')) + + def test_adds_missing_key_and_is_idempotent(self): + once = overrides.set_frontmatter("---\nname: x\n---\nbody\n", {"model": "Claude Sonnet 5"}) + self.assertEqual(once, '---\nname: x\nmodel: "Claude Sonnet 5"\n---\nbody\n') + self.assertEqual(overrides.set_frontmatter(once, {"model": "Claude Sonnet 5"}), once) + + def test_replaces_multi_line_values(self): + text = "---\nmodel:\n - a\n - b\ntools:\n- x\nname: n\n---\n" + result = overrides.set_frontmatter(text, {"model": "m", "tools": "t"}) + self.assertEqual(result, '---\nmodel: "m"\ntools: "t"\nname: n\n---\n') + + def test_keeps_crlf_line_endings(self): + result = overrides.set_frontmatter(AGENT.replace("\n", "\r\n"), {"model": "M", "extra": "E"}) + expected = AGENT.replace('"Claude Sonnet 4.6"', '"M"').replace('---\n\n', 'extra: "E"\n---\n\n') + self.assertEqual(result, expected.replace("\n", "\r\n")) + + def test_rejects_missing_or_ambiguous_frontmatter(self): + for text in ("no frontmatter\n", "---\nmodel: a\n", "---\nmodel: a\nmodel: b\n---\n"): + with self.subTest(text=text), self.assertRaises(overrides.OverrideError): + overrides.set_frontmatter(text, {"model": "x"}) + + +class ApplyTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + (self.root / "agent.md").write_text(AGENT, encoding="utf-8") + + def write_config(self, entries): + (self.root / overrides.CONFIG).write_text(json.dumps({"overrides": entries}), encoding="utf-8") + + def test_check_reports_drift_without_writing_then_apply_fixes_it(self): + self.write_config([{"file": "agent.md", "frontmatter": {"model": "Claude Sonnet 5"}}]) + self.assertEqual(overrides.main(["--check", "--root", str(self.root)]), 1) + self.assertEqual((self.root / "agent.md").read_text(encoding="utf-8"), AGENT) + self.assertEqual(overrides.main(["--root", str(self.root)]), 0) + self.assertEqual(overrides.main(["--check", "--root", str(self.root)]), 0) + self.assertIn('model: "Claude Sonnet 5"', (self.root / "agent.md").read_text(encoding="utf-8")) + + def test_fails_loudly_on_stale_or_invalid_entries(self): + for entries in ( + [{"file": "gone.md", "frontmatter": {"model": "x"}}], + [{"file": "../agent.md", "frontmatter": {"model": "x"}}], + [{"file": "agent.md", "frontmatter": {"model": 5}}], + [{"file": "agent.md", "frontmatter": {}}], + [{"frontmatter": {"model": "x"}}], + ): + with self.subTest(entries=entries): + self.write_config(entries) + with self.assertRaises(overrides.OverrideError): + overrides.apply(self.root) + self.assertEqual(overrides.main(["--root", str(self.root)]), 2) + + +class RepoTests(unittest.TestCase): + def test_committed_primitives_match_their_overrides(self): + self.assertEqual( + overrides.apply(ROOT, check=True), + [], + "a vendored file lost its local override; run " + "`python3 scripts/apply-apm-overrides.py` (see apm-overrides.json)", + ) + + +if __name__ == "__main__": + unittest.main() From fa7eb7c511b405806f1a8e1aa2aea12908126995 Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:54:23 +0100 Subject: [PATCH 2/3] fix: address review on the external primitives gate - Re-enable github/awesome-copilot digest updates (disabled org-wide), keep them out of the weekly non-major batch and allow missing git-refs timestamps, so the bump PR is actually opened. - Keep Renovate's security label through Label Sync. - Keep blank and comment lines inside a multi-line frontmatter value when applying overrides. Refs #210 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/labels.yaml | 5 +++++ docs/apm.md | 5 +++-- renovate.json5 | 7 +++++++ scripts/apply-apm-overrides.py | 10 +++++++--- tests/test_apm_overrides.py | 5 +++++ 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/labels.yaml b/.github/labels.yaml index 83573a9..25ad15f 100644 --- a/.github/labels.yaml +++ b/.github/labels.yaml @@ -10,6 +10,11 @@ color: "b60205" description: Changes third-party AI primitives; owner review required +# Renovate labels vulnerability-fix PRs `security` (shared base.json5). +- name: security + color: "ee0701" + description: Security fix + # GitHub's default labels, kept so the sync doesn't delete them. - name: dependencies color: "ededed" diff --git a/docs/apm.md b/docs/apm.md index 647e954..a86b427 100644 --- a/docs/apm.md +++ b/docs/apm.md @@ -189,8 +189,9 @@ flow below may auto-merge. 1. **Renovate bump.** Renovate's custom manager (see [`renovate.json5`](../renovate.json5)) bumps the pinned SHA. A package rule - turns off the shared auto-merge preset for `github/awesome-copilot` and adds - the `external-primitives` and `merge: manual` labels. The diff only shows a + re-enables these digest updates (off org-wide) in their own PR rather than + the weekly batch, turns off the shared auto-merge preset and adds the + `external-primitives` and `merge: manual` labels. The diff only shows a hash. Renovate can't remove the preset's `merge: auto` label, but `automerge: false` takes precedence. 2. **Materialize PR.** Merging the bump triggers diff --git a/renovate.json5 b/renovate.json5 index c8ffba4..1982f5a 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -39,8 +39,15 @@ // follow-up APM Materialize PR, so the owner reviews and merges both. // Renovate adds labels from every matching rule and can't remove one, // so the preset's `merge: auto` label stays; `automerge: false` wins. + // The shared presets also disable digest updates and batch non-major + // updates into one weekly PR: re-enable them here and keep this bump in + // its own PR. git-refs has no release timestamps, so the 14-day soak + // can't apply; without timestamp-optional the update stays pending. description: "Never auto-merge github/awesome-copilot bumps: third-party agent text needs owner review", matchDepNames: ["github/awesome-copilot"], + enabled: true, + groupName: null, + minimumReleaseAgeBehaviour: "timestamp-optional", automerge: false, commitMessageSuffix: "", addLabels: ["external-primitives", "merge: manual"], diff --git a/scripts/apply-apm-overrides.py b/scripts/apply-apm-overrides.py index 1ff7cd5..3faeaad 100755 --- a/scripts/apply-apm-overrides.py +++ b/scripts/apply-apm-overrides.py @@ -73,9 +73,13 @@ def set_frontmatter(text, values): end += 1 continue start = stop = matches[0] - # Swallow a multi-line value: indented lines or a column-0 `- ` list. - while stop + 1 < end and re.match(r"[ \t]|-([ \t]|$)", lines[stop + 1]): - stop += 1 + # Swallow a multi-line value: indented lines or a column-0 `- ` list, + # including blank and comment lines between its items. + for i in range(start + 1, end): + if lines[i].strip() and re.match(r"[ \t]|-([ \t]|$)", lines[i]): + stop = i + elif lines[i].strip() and not lines[i].startswith("#"): + break lines[start:stop + 1] = [line] end -= stop - start return "".join(lines) diff --git a/tests/test_apm_overrides.py b/tests/test_apm_overrides.py index e7046d1..8baa0ee 100644 --- a/tests/test_apm_overrides.py +++ b/tests/test_apm_overrides.py @@ -42,6 +42,11 @@ def test_replaces_multi_line_values(self): result = overrides.set_frontmatter(text, {"model": "m", "tools": "t"}) self.assertEqual(result, '---\nmodel: "m"\ntools: "t"\nname: n\n---\n') + def test_multi_line_values_may_contain_blank_and_comment_lines(self): + text = "---\nmodel:\n- a\n\n# note\n- b\n\n# keep\nname: n\n---\n" + result = overrides.set_frontmatter(text, {"model": "m"}) + self.assertEqual(result, '---\nmodel: "m"\n\n# keep\nname: n\n---\n') + def test_keeps_crlf_line_endings(self): result = overrides.set_frontmatter(AGENT.replace("\n", "\r\n"), {"model": "M", "extra": "E"}) expected = AGENT.replace('"Claude Sonnet 4.6"', '"M"').replace('---\n\n', 'extra: "E"\n---\n\n') From 8dfa40e63593ee5765c1d91e6b0b739c17030eaa Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:11:31 +0100 Subject: [PATCH 3/3] fix: reject multi-line override values and label reused materialize PRs - apply-apm-overrides now matches quoted keys and only replaces single-line values; other layouts fail loudly instead of producing invalid YAML that --check would accept. - APM Materialize adds the review labels to a reused PR, since the pinned open-pr only labels new PRs. Refs #210 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/apm-materialize.yml | 12 +++++++++ docs/apm.md | 4 ++- scripts/apply-apm-overrides.py | 36 ++++++++++++++++++--------- tests/test_apm_overrides.py | 27 +++++++++++++------- 4 files changed, 57 insertions(+), 22 deletions(-) diff --git a/.github/workflows/apm-materialize.yml b/.github/workflows/apm-materialize.yml index b3919b5..ffe4cd7 100644 --- a/.github/workflows/apm-materialize.yml +++ b/.github/workflows/apm-materialize.yml @@ -123,6 +123,7 @@ jobs: run: git fetch origin chore/apm-materialize || true - name: Open materialize PR + id: open-pr if: steps.precheck.outputs.skip != 'true' # renovate: datasource=github-tags depName=DevSecNinja/.github uses: DevSecNinja/.github/actions/open-pr@27fd66740c629ea38380dcd3bf93e7bc74c54247 # v1.9.0 @@ -157,3 +158,14 @@ jobs: `docs/apm.md`. Triggered by `${{ github.workflow }}` at commit `${{ github.sha }}`. + + - name: Label reused materialize PR + # The pinned open-pr only labels a PR it creates, so a PR left open by + # an earlier run would miss the review labels (DevSecNinja/.github#484 + # fixes this in the action). + if: steps.open-pr.outputs.created == 'false' && steps.open-pr.outputs.pr-number != '' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + PR_NUMBER: ${{ steps.open-pr.outputs.pr-number }} + run: |- + gh pr edit "${PR_NUMBER}" --add-label apm-sync --add-label external-primitives --add-label "merge: manual" diff --git a/docs/apm.md b/docs/apm.md index a86b427..64953bb 100644 --- a/docs/apm.md +++ b/docs/apm.md @@ -224,7 +224,9 @@ has no override setting). Add the change to } ``` -Each entry sets top-level frontmatter keys to string values. +Each entry sets top-level frontmatter keys to string values. Only a +single-line value is replaced; if upstream switches a key to a multi-line +value, the script fails and the override needs a manual update. [`scripts/apply-apm-overrides.py`](../scripts/apply-apm-overrides.py) applies them, and APM Materialize runs it after `apm install`. Run it yourself after a local `apm install`. It fails when a file is missing, so drop stale entries diff --git a/scripts/apply-apm-overrides.py b/scripts/apply-apm-overrides.py index 3faeaad..a396389 100755 --- a/scripts/apply-apm-overrides.py +++ b/scripts/apply-apm-overrides.py @@ -54,7 +54,12 @@ def load_overrides(root): def set_frontmatter(text, values): - """Return `text` with top-level frontmatter `values` set; add missing keys.""" + """Return `text` with top-level frontmatter `values` set; add missing keys. + + Only a single-line value is replaced. Any other layout (block or flow + collection, block scalar, continued plain scalar) raises instead of + risking invalid YAML; update such an override by hand. + """ lines = text.splitlines(keepends=True) if not lines or lines[0].rstrip("\r\n") != "---": raise OverrideError("no YAML frontmatter") @@ -62,9 +67,10 @@ def set_frontmatter(text, values): if end is None: raise OverrideError("unterminated YAML frontmatter") newline = "\r\n" if lines[0].endswith("\r\n") else "\n" + top_level_key = re.compile(r"""(?:[A-Za-z0-9_-]+|'[^']*'|"[^"]*")[ \t]*:(?:[ \t]|$)""") for key, value in values.items(): line = f"{key}: {json.dumps(value, ensure_ascii=False)}{newline}" - pattern = re.compile(rf"{re.escape(key)}[ \t]*:") + pattern = re.compile(rf"""(?:{re.escape(key)}|'{re.escape(key)}'|"{re.escape(key)}")[ \t]*:(?:[ \t]|$)""") matches = [i for i in range(1, end) if pattern.match(lines[i])] if len(matches) > 1: raise OverrideError(f"frontmatter key `{key}` appears more than once") @@ -72,16 +78,22 @@ def set_frontmatter(text, values): lines.insert(end, line) end += 1 continue - start = stop = matches[0] - # Swallow a multi-line value: indented lines or a column-0 `- ` list, - # including blank and comment lines between its items. - for i in range(start + 1, end): - if lines[i].strip() and re.match(r"[ \t]|-([ \t]|$)", lines[i]): - stop = i - elif lines[i].strip() and not lines[i].startswith("#"): - break - lines[start:stop + 1] = [line] - end -= stop - start + index = matches[0] + current = lines[index].split(":", 1)[1].strip() + following = next( + (candidate for candidate in lines[index + 1:end] if candidate.strip() and not candidate.startswith("#")), + None, + ) + multi_line = ( + not current + or current[0] in "|>" + or (current[0] == "[" and not current.rstrip().endswith("]")) + or (current[0] == "{" and not current.rstrip().endswith("}")) + or (following is not None and not top_level_key.match(following)) + ) + if multi_line: + raise OverrideError(f"frontmatter key `{key}` has a multi-line value; update it by hand") + lines[index] = line return "".join(lines) diff --git a/tests/test_apm_overrides.py b/tests/test_apm_overrides.py index 8baa0ee..3984cef 100644 --- a/tests/test_apm_overrides.py +++ b/tests/test_apm_overrides.py @@ -37,15 +37,24 @@ def test_adds_missing_key_and_is_idempotent(self): self.assertEqual(once, '---\nname: x\nmodel: "Claude Sonnet 5"\n---\nbody\n') self.assertEqual(overrides.set_frontmatter(once, {"model": "Claude Sonnet 5"}), once) - def test_replaces_multi_line_values(self): - text = "---\nmodel:\n - a\n - b\ntools:\n- x\nname: n\n---\n" - result = overrides.set_frontmatter(text, {"model": "m", "tools": "t"}) - self.assertEqual(result, '---\nmodel: "m"\ntools: "t"\nname: n\n---\n') - - def test_multi_line_values_may_contain_blank_and_comment_lines(self): - text = "---\nmodel:\n- a\n\n# note\n- b\n\n# keep\nname: n\n---\n" - result = overrides.set_frontmatter(text, {"model": "m"}) - self.assertEqual(result, '---\nmodel: "m"\n\n# keep\nname: n\n---\n') + def test_replaces_quoted_keys_without_duplicating_them(self): + for key in ("'model'", '"model"'): + with self.subTest(key=key): + text = f"---\n{key}: 'Claude Sonnet 4.6'\nname: n\n---\n" + result = overrides.set_frontmatter(text, {"model": "m"}) + self.assertEqual(result, '---\nmodel: "m"\nname: n\n---\n') + + def test_rejects_multi_line_values_instead_of_corrupting_them(self): + for value in ( + "model:\n - a\n - b\n", + "model:\n- a\n\n# note\n- b\n", + 'model: [\n "upstream"\n]\n', + "model: {a: 1,\n b: 2}\n", + "model: |\n text\n", + "model: plain\n continued\n", + ): + with self.subTest(value=value), self.assertRaises(overrides.OverrideError): + overrides.set_frontmatter(f"---\n{value}name: n\n---\n", {"model": "m"}) def test_keeps_crlf_line_endings(self): result = overrides.set_frontmatter(AGENT.replace("\n", "\r\n"), {"model": "M", "extra": "E"})