From 6a54273ac9fae715d712c9a1095ab10f51822dda Mon Sep 17 00:00:00 2001 From: aadimch Date: Mon, 9 Nov 2026 04:38:18 +0000 Subject: [PATCH 1/6] feat(ci): refuse YAML that other parsers read differently --- .claude/CLAUDE.md | 4 +- .../skill-rules/conformance-cases.json | 134 ++- .../skill-rules/yaml-conformance-cases.json | 855 ++++++++++++++++++ .github/scripts/skill_checks.py | 477 ++++++++-- .github/scripts/validate_skills.py | 92 +- .github/workflows/validate-skills-main.yml | 1 + .github/workflows/validate-skills.yml | 1 + .kiro/steering/project-conventions.md | 4 +- CONTRIBUTING.md | 6 +- tests/test_skill_checks.py | 111 ++- tests/test_validate_skills.py | 67 +- 11 files changed, 1565 insertions(+), 187 deletions(-) create mode 100644 .github/scripts/skill-rules/yaml-conformance-cases.json diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index bef0f8eca..54d1f1ed6 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -92,7 +92,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar `.github/workflows/validate-skills.yml` runs `.github/scripts/validate_skills.py` on every pull request, and `.github/workflows/validate-skills-main.yml` runs it again after each push to `main`. `.github/workflows/build-docs.yml` runs `mkdocs build --strict` on pull requests and on `main`. The rules and their reasons are in the "Skill Publishing Rules" section of [CONTRIBUTING.md](../CONTRIBUTING.md). -- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, and a deprecated skill still listed in a `connectors/*/connector.yaml`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. +- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. - Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. @@ -101,7 +101,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. - `check_added_content` warns about internal host or tool names only on lines a PR adds, like `scan-aws-identifiers`, so names already on `main` don't warn on every PR. - With `--base-ref`, `check_history` compares each skill with its merge-base copy: no removed folders, no reuse of a path deleted in `main`'s first-parent history (`git log --first-parent --no-renames --diff-filter=D`, skipped with a warning on a shallow clone), a higher version whenever `content_hash` of the published files changes, one step at a time (next patch, minor or major), and `1.0.0` for a new skill. No folder is ever deleted, not even in an emergency: deprecate it instead. The hash function is pinned by the golden fixture under `content_hash` in `conformance-cases.json`, so any other implementation can prove it computes the same hash. -- With or without a base, a deprecated skill can't stay in any `connectors/*/connector.yaml` `skills:` list (`connector_skill_references`, which composes the YAML rather than constructing it). A `connector.yaml` that can't be read, isn't a mapping, has a `skills:` value that isn't a list of skill names, or writes `skills:` more than once gets a warning; the names it does list are still checked. +- With or without a base, a deprecated skill can't stay in any `connectors/*/connector.yaml` `skills:` list (`connector_skill_references`). Every `connectors/*/connector.yaml` and `connectors/releases.yaml` is read with `skill_checks.read_yaml`, the reader the frontmatter uses, so a file that isn't UTF-8 or breaks a YAML rule is an error; the cases are in `skill-rules/yaml-conformance-cases.json`, which `tests/test_skill_checks.py` runs. A `connector.yaml` that isn't a mapping or has a `skills:` value that isn't a list of skill names gets a warning; the names it does list are still checked. - Secrets in test fixtures are built at runtime (`"AKIA" + ...`) or stored as base64 in the conformance cases, so no file in the repository contains a literal key or invisible character. Internal names in tests are made up (`corp.example`, `Rivendell`), never real hosts or tools. - `conformance-cases.json` holds language-neutral cases for these rules. Another tool that validates or publishes these skills should give the same `expect` result on the cases for the rules it implements; some rules here are stricter than any other tool's. The script runs every case before checking any skill and exits 2 on a mismatch. A rule change needs a matching case change in the same PR. - Every rule applies to every skill; nothing is exempted. A rule that existing skills don't meet yet starts as a warning and becomes an error after a cleanup PR brings every skill into line: a missing `metadata.summary`, and dimension values outside the vocabulary. A missing `metadata.aws-devops-agent-skills.agent-types` stays a warning. diff --git a/.github/scripts/skill-rules/conformance-cases.json b/.github/scripts/skill-rules/conformance-cases.json index d8ce5c090..329ec505a 100644 --- a/.github/scripts/skill-rules/conformance-cases.json +++ b/.github/scripts/skill-rules/conformance-cases.json @@ -1882,13 +1882,45 @@ "id": "value-other-parsers-read-as-a-number", "name": "demo", "expect": "fail", - "error": "other YAML parsers read", + "error": "don't all read as the same type", "files": { "SKILL.md": { "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\n summary: 1e3\n---\n\n# Demo\n" } } }, + { + "id": "value-read-as-another-type-nested", + "name": "demo", + "expect": "fail", + "error": "don't all read as the same type", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\nextra:\n enabled: yes\n---\n\n# Demo\n" + } + } + }, + { + "id": "value-short-yaml11-bool", + "name": "demo", + "expect": "fail", + "error": "don't all read as the same type", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\ntitle: n\nmetadata:\n author: octocat\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + } + }, + { + "id": "value-typed-alike-by-every-parser", + "name": "demo", + "expect": "pass", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\nextra:\n a: true\n b: null\n c: 3\n d: 1.5\n e: 0.0001\n f:\n---\n\n# Demo\n" + } + } + }, { "id": "version-leading-zero", "name": "demo", @@ -1966,6 +1998,49 @@ } } }, + { + "id": "yaml-block-scalar-indentation-indicator", + "name": "demo", + "expect": "fail", + "error": "indentation indicator", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: |2\n Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + } + }, + { + "id": "yaml-block-scalar-last-strip-chomped", + "name": "demo", + "expect": "pass", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\ncompatibility: |-\n Requires nothing.\n---\n\n# Demo\n" + } + } + }, + { + "id": "yaml-block-scalar-last-without-line-break", + "name": "demo", + "expect": "fail", + "error": "no line break after it", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\ncompatibility: |\n Requires nothing.\n---\n\n# Demo\n" + } + } + }, + { + "id": "yaml-carriage-return", + "name": "demo", + "expect": "fail", + "error": "carriage return without a line feed", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\r deprecated: \"true\"\n---\n\n# Demo\n" + } + } + }, { "id": "yaml-directive", "name": "demo", @@ -1981,7 +2056,7 @@ "id": "yaml-document-end-marker", "name": "demo", "expect": "fail", - "error": "document markers", + "error": "document end markers", "files": { "SKILL.md": { "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\n...\n---\n\n# Demo\n" @@ -2010,6 +2085,28 @@ } } }, + { + "id": "yaml-escaped-line-break-before-blank-line", + "name": "demo", + "expect": "fail", + "error": "escaped line break before a blank line", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: \"Use this skill when testing\\\n\n the publishing rules.\"\nmetadata:\n author: octocat\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + } + }, + { + "id": "yaml-escaped-surrogate", + "name": "demo", + "expect": "fail", + "error": "UTF-16 surrogate", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: \"Use this skill when testing \\uD83D\\uDE00 the publishing rules.\"\nmetadata:\n author: octocat\n version: \"1.0.0\"\n---\n\n# Demo\n" + } + } + }, { "id": "yaml-flow-collection", "name": "demo", @@ -2032,6 +2129,17 @@ } } }, + { + "id": "yaml-key-short-yaml11-bool", + "name": "demo", + "expect": "fail", + "error": "keys must be plain text", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\nextra:\n y: one\n---\n\n# Demo\n" + } + } + }, { "id": "yaml-line-separator", "name": "demo", @@ -2043,6 +2151,28 @@ } } }, + { + "id": "yaml-merge-key-quoted", + "name": "demo", + "expect": "fail", + "error": "<< key", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\nextra:\n \"<<\": one\n---\n\n# Demo\n" + } + } + }, + { + "id": "yaml-nested-too-deeply", + "name": "demo", + "expect": "fail", + "error": "nested too deeply", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\nextra:\n k1:\n k2:\n k3:\n k4:\n k5:\n k6:\n k7:\n k8:\n k9:\n k10:\n k11:\n k12:\n k13:\n k14:\n k15:\n k16:\n k17:\n k18:\n k19:\n k20:\n k21:\n k22:\n k23:\n k24:\n k25:\n k26:\n k27:\n k28:\n k29:\n k30:\n k31:\n k32:\n k33:\n k34:\n k35:\n k36:\n k37:\n k38:\n k39:\n k40:\n k41:\n k42:\n k43:\n k44:\n k45:\n k46:\n k47:\n k48:\n k49:\n k50:\n k51:\n k52:\n k53:\n k54:\n k55:\n k56:\n k57:\n k58:\n k59:\n k60:\n k61:\n k62:\n k63:\n k64:\n k: v\n---\n\n# Demo\n" + } + } + }, { "id": "yaml-tab", "name": "demo", diff --git a/.github/scripts/skill-rules/yaml-conformance-cases.json b/.github/scripts/skill-rules/yaml-conformance-cases.json new file mode 100644 index 000000000..eeb2735c9 --- /dev/null +++ b/.github/scripts/skill-rules/yaml-conformance-cases.json @@ -0,0 +1,855 @@ +{ + "description": "Conformance cases for the YAML in connectors//connector.yaml and connectors/releases.yaml. Other YAML parsers read some YAML differently from PyYAML, or refuse it, so these files may hold only YAML that every parser reads the same way. Each case is a whole file and gives its expect result: pass means the YAML is read without a problem, fail means it is refused, and error is a fragment of this repository's own message. Another tool that reads these files should give the same result on every case. The SKILL.md frontmatter follows the same rules and stricter ones (see conformance-cases.json), so a pass case here can fail as frontmatter, and a frontmatter never ends in a line break, since the closing --- line holds it. tests/test_skill_checks.py runs every case.", + "cases": [ + { + "id": "plain strings", + "yaml": "name: cribl\nversion: 1.0.0\ntitle: yes please\nteam: on-call\nurl: https://example.com/a?b=c\n", + "expect": "pass" + }, + { + "id": "quoted values that a parser would type if plain", + "yaml": "a: \"1.0\"\nb: 'yes'\nc: \"2026-11-08\"\nd: \"~\"\ne: \"<<\"\nf: \"0x1F\"\n", + "expect": "pass" + }, + { + "id": "canonical ints", + "yaml": "a: 0\nb: 3\nc: -3\nd: 9007199254740991\ne: -9007199254740991\n", + "expect": "pass" + }, + { + "id": "decimals that round-trip", + "yaml": "a: 1.5\nb: -0.25\nc: 0.0001\n", + "expect": "pass" + }, + { + "id": "true, false and null", + "yaml": "a: true\nb: false\nc: null\nd: ~\ne:\n", + "expect": "pass" + }, + { + "id": "quoted keys", + "yaml": "'true': a\n\"1\": b\n'null': c\n\"1.0\": d\n'~': e\n", + "expect": "pass" + }, + { + "id": "flow collections", + "yaml": "a: [x, 1, {b: true}]\n", + "expect": "pass" + }, + { + "id": "block scalars", + "yaml": "a: |\n yes\nb: >\n 1.0\n", + "expect": "pass" + }, + { + "id": "CRLF line ends", + "yaml": "a: 1\r\nb: x\r\n", + "expect": "pass" + }, + { + "id": "a document start marker", + "yaml": "---\na: 1\n", + "expect": "pass" + }, + { + "id": "tag !!str on a value", + "yaml": "a: !!str 1\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "tag !!map on a collection", + "yaml": "a: !!map {b: 1}\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "tag on a key", + "yaml": "!!str k: v\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "tag on an explicit key", + "yaml": "? !!str k\n: v\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "tag on a key in a flow map", + "yaml": "{!!str k: v}\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "non-specific tag !", + "yaml": "a: ! x\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "custom tag", + "yaml": "a: !custom x\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "tag on an empty node", + "yaml": "a: !!null\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "tag on the root collection", + "yaml": "!!map\nk: v\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "tag on a sequence item", + "yaml": "- !!str x\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "tag on a block scalar", + "yaml": "a: !!str |\n x\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "tag in a flow sequence", + "yaml": "a: [!!str x]\n", + "expect": "fail", + "error": "uses tags" + }, + { + "id": "merge key with an alias", + "yaml": "base: &b {x: 1}\nchild:\n <<: *b\n", + "expect": "fail", + "error": "uses anchors" + }, + { + "id": "merge key without an alias", + "yaml": "child:\n <<: {x: 1}\n", + "expect": "fail", + "error": "<< key" + }, + { + "id": "single-quoted merge key", + "yaml": "'<<': {x: 1}\n", + "expect": "fail", + "error": "<< key" + }, + { + "id": "double-quoted merge key", + "yaml": "\"<<\": {x: 1}\n", + "expect": "fail", + "error": "<< key" + }, + { + "id": "merge key in a flow map", + "yaml": "{<<: {x: 1}}\n", + "expect": "fail", + "error": "<< key" + }, + { + "id": "explicit merge key", + "yaml": "? <<\n: {x: 1}\n", + "expect": "fail", + "error": "<< key" + }, + { + "id": "byte order mark at the start", + "yaml": "\ufeffa: 1\n", + "expect": "fail", + "error": "byte order mark" + }, + { + "id": "byte order mark in a plain value", + "yaml": "a: x\ufeffy\n", + "expect": "fail", + "error": "byte order mark" + }, + { + "id": "byte order mark in a quoted value", + "yaml": "a: \"x\ufeff\"\n", + "expect": "fail", + "error": "byte order mark" + }, + { + "id": "byte order mark in a comment", + "yaml": "# c\ufeff\na: 1\n", + "expect": "fail", + "error": "byte order mark" + }, + { + "id": "plain value yes", + "yaml": "a: yes\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value No", + "yaml": "a: No\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value ON", + "yaml": "a: ON\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value True", + "yaml": "a: True\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value NULL", + "yaml": "a: NULL\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value y", + "yaml": "a: y\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value =", + "yaml": "a: =\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value <<", + "yaml": "a: <<\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 1.0", + "yaml": "a: 1.0\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 1.10", + "yaml": "a: 1.10\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 010", + "yaml": "a: 010\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 0x1F", + "yaml": "a: 0x1F\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 0o7", + "yaml": "a: 0o7\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value -0o7", + "yaml": "a: -0o7\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 0b1", + "yaml": "a: 0b1\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 1_000", + "yaml": "a: 1_000\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value +1", + "yaml": "a: +1\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value -0", + "yaml": "a: -0\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 1e3", + "yaml": "a: 1e3\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value .5", + "yaml": "a: .5\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value .inf", + "yaml": "a: .inf\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value -.inf", + "yaml": "a: -.inf\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value .nan", + "yaml": "a: .nan\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 2026-11-08", + "yaml": "a: 2026-11-08\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 2026-11-08T10:00:00Z", + "yaml": "a: 2026-11-08T10:00:00Z\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 2026-11-08 10:00:00", + "yaml": "a: 2026-11-08 10:00:00\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 1:30", + "yaml": "a: 1:30\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 0.00001", + "yaml": "a: 0.00001\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value 9007199254740993", + "yaml": "a: 9007199254740993\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain value yes in a flow sequence", + "yaml": "a: [x, yes]\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain timestamp folded over two lines", + "yaml": "a: 2026-11-08\n 10:00:00\n", + "expect": "fail", + "error": "don't all read as the same type" + }, + { + "id": "plain key true", + "yaml": "true: a\n", + "expect": "fail", + "error": "keys must be plain text" + }, + { + "id": "plain key null", + "yaml": "null: a\n", + "expect": "fail", + "error": "keys must be plain text" + }, + { + "id": "plain key ~", + "yaml": "~: a\n", + "expect": "fail", + "error": "keys must be plain text" + }, + { + "id": "plain key 1", + "yaml": "1: a\n", + "expect": "fail", + "error": "keys must be plain text" + }, + { + "id": "plain key 1.5", + "yaml": "1.5: a\n", + "expect": "fail", + "error": "keys must be plain text" + }, + { + "id": "plain key yes", + "yaml": "yes: a\n", + "expect": "fail", + "error": "keys must be plain text" + }, + { + "id": "plain key 1e3", + "yaml": "1e3: a\n", + "expect": "fail", + "error": "keys must be plain text" + }, + { + "id": "plain key =", + "yaml": "=: a\n", + "expect": "fail", + "error": "keys must be plain text" + }, + { + "id": "empty plain key", + "yaml": ": a\n", + "expect": "fail", + "error": "not valid YAML" + }, + { + "id": "duplicate keys", + "yaml": "a: 1\na: 2\n", + "expect": "fail", + "error": "duplicate key" + }, + { + "id": "duplicate keys, one quoted", + "yaml": "a: 1\n'a': 2\n", + "expect": "fail", + "error": "duplicate key" + }, + { + "id": "anchor", + "yaml": "a: &x 1\n", + "expect": "fail", + "error": "uses anchors" + }, + { + "id": "anchor and alias", + "yaml": "a: &x 1\nb: *x\n", + "expect": "fail", + "error": "uses anchors" + }, + { + "id": "%YAML directive", + "yaml": "%YAML 1.1\n---\na: 1\n", + "expect": "fail", + "error": "uses directives" + }, + { + "id": "%TAG directive", + "yaml": "%TAG !e! tag:example.com,2000:\n---\na: 1\n", + "expect": "fail", + "error": "uses directives" + }, + { + "id": "multiple documents", + "yaml": "a: 1\n---\nb: 2\n", + "expect": "fail", + "error": "not valid YAML" + }, + { + "id": "document end marker", + "yaml": "a: 1\n...\n", + "expect": "fail", + "error": "uses document end markers" + }, + { + "id": "document end marker with a comment", + "yaml": "a: 1\n... # comment\n", + "expect": "fail", + "error": "uses document end markers" + }, + { + "id": "sequence as a key", + "yaml": "? [k]\n: v\n", + "expect": "fail", + "error": "keys must be plain text" + }, + { + "id": "map as a key", + "yaml": "? {k: 1}\n: v\n", + "expect": "fail", + "error": "keys must be plain text" + }, + { + "id": "tab after a colon", + "yaml": "a:\tb\n", + "expect": "fail", + "error": "not valid YAML" + }, + { + "id": "tab as indentation", + "yaml": "a:\n\tb: 1\n", + "expect": "fail", + "error": "not valid YAML" + }, + { + "id": "carriage return without a line feed", + "yaml": "a: 1\rb: 2\n", + "expect": "fail", + "error": "carriage return without a line feed" + }, + { + "id": "U+2028 in a comment", + "yaml": "# c\u2028b: 2\na: 1\n", + "expect": "fail", + "error": "U+2028" + }, + { + "id": "nesting 65 levels deep", + "yaml": "[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[x]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]\n", + "expect": "fail", + "error": "nested too deeply" + }, + { + "id": "literal block scalar with no line break after it", + "yaml": "name: x\ndescription: |\n Does a thing.", + "expect": "fail", + "error": "no line break after it" + }, + { + "id": "folded block scalar with no line break after it", + "yaml": "name: x\ndescription: >\n Does a thing.", + "expect": "fail", + "error": "no line break after it" + }, + { + "id": "keep block scalar with no line break after it", + "yaml": "name: x\ndescription: |+\n Does a thing.", + "expect": "fail", + "error": "no line break after it" + }, + { + "id": "block scalar indentation indicator before a longer blank line", + "yaml": "a: |1\n x\n \n", + "expect": "fail", + "error": "indentation indicator" + }, + { + "id": "folded block scalar indentation indicator", + "yaml": "a: >1\n x\n \n", + "expect": "fail", + "error": "indentation indicator" + }, + { + "id": "flow map key that starts with ?", + "yaml": "{?a: b}\n", + "expect": "fail", + "error": "unquoted entry in a flow collection" + }, + { + "id": "flow sequence item that starts with ?", + "yaml": "[?b]\n", + "expect": "fail", + "error": "unquoted entry in a flow collection" + }, + { + "id": "flow map key that is only ?", + "yaml": "{?: v}\n", + "expect": "fail", + "error": "unquoted entry in a flow collection" + }, + { + "id": "flow sequence item that holds ?", + "yaml": "k: [a?b]\n", + "expect": "fail", + "error": "unquoted entry in a flow collection" + }, + { + "id": "flow map value that starts with ?", + "yaml": "{a: ?b}\n", + "expect": "fail", + "error": "unquoted entry in a flow collection" + }, + { + "id": "flow map value that ends with ?", + "yaml": "k: {a: b?}\n", + "expect": "fail", + "error": "unquoted entry in a flow collection" + }, + { + "id": "flow sequence item that starts with :", + "yaml": "k: [:b]\n", + "expect": "fail", + "error": "not valid YAML" + }, + { + "id": "flow map key that starts with :", + "yaml": "{:a}\n", + "expect": "fail", + "error": "not valid YAML" + }, + { + "id": "flow plain scalar folded onto a ?", + "yaml": "k: [a\n ?b]\n", + "expect": "fail", + "error": "unquoted entry in a flow collection" + }, + { + "id": "escaped surrogate value", + "yaml": "a: \"\\uD800\"\n", + "expect": "fail", + "error": "UTF-16 surrogate" + }, + { + "id": "escaped surrogate key", + "yaml": "\"\\uDC00x\": v\n", + "expect": "fail", + "error": "UTF-16 surrogate" + }, + { + "id": "escaped surrogate pair", + "yaml": "a: \"\\uD83D\\uDE00\"\n", + "expect": "fail", + "error": "UTF-16 surrogate" + }, + { + "id": "escaped surrogate in a \\U escape", + "yaml": "a: \"\\U0000D800\"\n", + "expect": "fail", + "error": "UTF-16 surrogate" + }, + { + "id": "escaped surrogate after an escaped backslash", + "yaml": "a: \"\\\\\\uD800\"\n", + "expect": "fail", + "error": "UTF-16 surrogate" + }, + { + "id": "escaped line break before a blank line", + "yaml": "a: \"x\\\n\n y\"\n", + "expect": "fail", + "error": "escaped line break before a blank line" + }, + { + "id": "quoted flow map key over two lines", + "yaml": "k: {\"a\n b\": v}\n", + "expect": "fail", + "error": "over more than one line" + }, + { + "id": "plain flow map key over two lines", + "yaml": "k: {a\n b: v}\n", + "expect": "fail", + "error": "over more than one line" + }, + { + "id": "multi-line flow: double-quoted key before a colon on the next line", + "yaml": "k: {\"a\"\n : v}\n", + "expect": "fail", + "error": "over more than one line" + }, + { + "id": "multi-line flow: plain key before a colon on the next line", + "yaml": "k: {a\n : v}\n", + "expect": "fail", + "error": "over more than one line" + }, + { + "id": "multi-line flow: single-quoted key before a colon on the next line", + "yaml": "k: {'a'\n : v}\n", + "expect": "fail", + "error": "over more than one line" + }, + { + "id": "multi-line flow: root map with the colon on the next line", + "yaml": "{\"a\"\n: v}\n", + "expect": "fail", + "error": "over more than one line" + }, + { + "id": "multi-line flow: second entry with the colon on the next line", + "yaml": "k: {a: b,\n c\n : d}\n", + "expect": "fail", + "error": "over more than one line" + }, + { + "id": "multi-line flow: comment before the colon", + "yaml": "k: {\"a\" # c\n : v}\n", + "expect": "fail", + "error": "over more than one line" + }, + { + "id": "multi-line flow: sequence", + "yaml": "k: [a,\n b]\n", + "expect": "fail", + "error": "over more than one line" + }, + { + "id": "multi-line flow: map", + "yaml": "k: {a: 1,\n b: 2}\n", + "expect": "fail", + "error": "over more than one line" + }, + { + "id": "strip-chomped block scalars with no line break after them", + "yaml": "a: x\nb: |-\n y\nc: >-\n z\n w", + "expect": "pass" + }, + { + "id": "strip-chomped block scalar after trailing blank lines", + "yaml": "a: |-\n y\n\n ", + "expect": "pass" + }, + { + "id": "escapes that every parser decodes the same way", + "yaml": "a: \"\\u00e9\\U0001F600\\x41\\t\\N\"\n", + "expect": "pass" + }, + { + "id": "escaped backslash before u", + "yaml": "a: \"\\\\uD800\"\n", + "expect": "pass" + }, + { + "id": "escaped line break inside a line", + "yaml": "a: \"x\\\n y\"\n", + "expect": "pass" + }, + { + "id": "block scalar followed by a line break", + "yaml": "a: |\n x\nb: >-\n y\n z\n", + "expect": "pass" + }, + { + "id": "empty block scalar at the end", + "yaml": "a: |", + "expect": "pass" + }, + { + "id": "? and : in block plain scalars", + "yaml": "a: a?b\nb: ?b\nc: :b\n", + "expect": "pass" + }, + { + "id": "quoted ? and plain : in flow collections", + "yaml": "k: [\"a?b\", a:b, -a]\n", + "expect": "pass" + }, + { + "id": "block scalar whose first line is indented less than a blank line before it", + "yaml": "a: |\n \n # c\nb: 1\n", + "expect": "fail", + "error": "indented less than a blank line before it" + }, + { + "id": "block scalar after a blank line no longer than its first line", + "yaml": "a: |\n \n # c\n", + "expect": "pass" + }, + { + "id": "block scalar after a long blank line, then a key at the parent's indentation", + "yaml": "a:\n b: |\n \n c: 1\n", + "expect": "pass" + }, + { + "id": "quoted value whose next line is not indented past its key", + "yaml": "k:\n j: \"a\n b\"\n", + "expect": "fail", + "error": "isn't indented past its key or list item" + }, + { + "id": "quoted value that closes at the start of a line", + "yaml": "k: '\n'\n", + "expect": "fail", + "error": "isn't indented past its key or list item" + }, + { + "id": "quoted value whose next line is indented past its key", + "yaml": "k:\n j: \"a\n b\"\n", + "expect": "pass" + }, + { + "id": "tab before the indentation of a quoted value's next line", + "yaml": "k: \"a\n\t b\"\n", + "expect": "fail", + "error": "isn't indented past its key or list item" + }, + { + "id": "tab after the indentation of a quoted value's next line", + "yaml": "k: \"a\n \tb\"\n", + "expect": "pass" + }, + { + "id": "block scalar that ends in a line of spaces with no line break", + "yaml": "a: |\n ", + "expect": "fail", + "error": "no line break after it" + }, + { + "id": "keep block scalar that ends in a line of spaces as indented as its content", + "yaml": "a: |+\n x\n ", + "expect": "fail", + "error": "no line break after it" + }, + { + "id": "block scalar that ends in a line of spaces as indented as its content", + "yaml": "a: |\n x\n ", + "expect": "pass" + }, + { + "id": "lone - key with no value in a flow map", + "yaml": "k: {-}\n", + "expect": "fail", + "error": "unquoted entry in a flow collection" + }, + { + "id": "lone - value in a flow sequence", + "yaml": "k: [a, -]\n", + "expect": "fail", + "error": "unquoted entry in a flow collection" + }, + { + "id": "lone - key before a colon in a flow map", + "yaml": "k: {-: v}\n", + "expect": "pass" + }, + { + "id": "comment right after a double-quoted value", + "yaml": "k: \"a\"#c\n", + "expect": "fail", + "error": "right after a closing quote or bracket" + }, + { + "id": "comment right after a single-quoted value", + "yaml": "k: 'a'#c\n", + "expect": "fail", + "error": "right after a closing quote or bracket" + }, + { + "id": "comment right after a flow sequence", + "yaml": "k: [a]#c\n", + "expect": "fail", + "error": "right after a closing quote or bracket" + }, + { + "id": "comment right after a flow map", + "yaml": "k: {a: b}#c\n", + "expect": "fail", + "error": "right after a closing quote or bracket" + }, + { + "id": "comment after a space after a quoted value", + "yaml": "k: \"a\" #c\n", + "expect": "pass" + } + ] +} diff --git a/.github/scripts/skill_checks.py b/.github/scripts/skill_checks.py index a8b0dc73e..50943bf35 100644 --- a/.github/scripts/skill_checks.py +++ b/.github/scripts/skill_checks.py @@ -188,8 +188,51 @@ ) # Characters PyYAML reads as line breaks and YAML 1.2 parsers don't. YAML11_LINE_BREAK_PATTERN = re.compile("[\x85\u2028\u2029]") +# Plain values that PyYAML's resolver (with the y and n of the YAML 1.1 bool +# type) or the YAML 1.2 core schema reads as something other than text. +NON_TEXT_PLAIN_SCALAR_PATTERNS = ( + re.compile(r"[yYnN]|yes|Yes|YES|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF"), + re.compile(r"~|null|Null|NULL|"), + re.compile(r"<<|="), + re.compile(r"[-+]?(?:0b[0-1_]+|0[0-7_]+|0|[1-9][0-9_]*|0x[0-9a-fA-F_]+|[1-9][0-9_]*(?::[0-5]?[0-9])+)"), + re.compile( + r"[-+]?[0-9][0-9_]*\.[0-9_]*(?:[eE][-+][0-9]+)?|\.[0-9][0-9_]*(?:[eE][-+][0-9]+)?" + r"|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN)" + ), + re.compile( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}|[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}(?:[Tt]|[ \t]+)[0-9]{1,2}:[0-9]{2}:[0-9]{2}" + r"(?:\.[0-9]*)?(?:[ \t]*(?:Z|[-+][0-9]{1,2}(?::[0-9]{2})?))?" + ), + re.compile(r"[-+]?[0-9]+|[-+]?0o[0-7]+|0x[0-9a-fA-F]+"), + re.compile(r"[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN)"), + YAML12_NUMBER_PATTERN, +) +CANONICAL_INTEGER_PATTERN = re.compile(r"-?(?:0|[1-9][0-9]*)") +CANONICAL_DECIMAL_PATTERN = re.compile(r"-?(?:0|[1-9][0-9]*)\.[0-9]+") +# A JavaScript number holds integers only up to 2**53, and Python writes a +# decimal below 1e-4 with an exponent, so neither keeps its text as JSON. +MAX_SAFE_INTEGER = 2**53 - 1 +MIN_DECIMAL_MAGNITUDE = 1e-4 +# PyYAML ends a plain scalar at a ? in a flow collection, and reads a ? or : at its start as an indicator. +FLOW_PLAIN_SCALAR_PATTERN = re.compile(r"^[?:]|\?") +FLOW_PLAIN_SCALAR_PROBLEM = ( + "has an unquoted entry in a flow collection that starts with ? or :, holds a ?, or is a lone -, which YAML " + "parsers read differently; put it in quotes" +) +# Each escape is matched once, so an escaped backslash can't start another escape. +DOUBLE_QUOTED_ESCAPE_PATTERN = re.compile(r"\\(?:u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8})|(\r?\n[ \t]*\r?\n)|.)", re.S) +BLOCK_SCALAR_INDICATORS_PATTERN = re.compile(r"[|>][-+0-9]*") +# The blank lines after a block scalar's header, then the indentation of the first line that isn't blank. +BLOCK_SCALAR_LEADING_LINES_PATTERN = re.compile(r"((?: *\r?\n)*)( *)[^ \r\n]") +MAX_YAML_DEPTH = 64 +YAML_TEXT_PROBLEMS = ( + (YAML11_LINE_BREAK_PATTERN, "U+{code:04X}, which YAML parsers disagree on as a line break; use a newline"), + (re.compile(r"\r(?!\n)"), "a carriage return without a line feed, which PyYAML reads as a line break; use a newline"), + (re.compile("\ufeff"), "a byte order mark (U+FEFF), which YAML parsers don't all read the same way; remove it"), +) FRONTMATTER_PATTERN = re.compile(r"\A---\r?\n(?:(.*?)\r?\n)?---[ \t]*(?:\r?\n|\Z)", re.S) +FRONTMATTER_FIRST_LINE = 2 STR_TAG = "tag:yaml.org,2002:str" _SCALAR_KINDS = { @@ -201,16 +244,18 @@ } # YAML features that different parsers read differently, or that let one part -# of the frontmatter silently change another. Other tools parse the same text, -# in other languages, so the frontmatter is limited to plain block mappings and -# scalars that every parser reads the same way. -_FORBIDDEN_TOKENS = ( +# of a file silently change another. Other tools parse the same text, in other +# languages. +YAML_FORBIDDEN_TOKENS = ( (yaml.AnchorToken, "anchors (&)"), (yaml.AliasToken, "aliases (*)"), (yaml.TagToken, "tags (! or !!)"), (yaml.DirectiveToken, "directives (%)"), - (yaml.DocumentStartToken, "document markers (--- or ...)"), - (yaml.DocumentEndToken, "document markers (--- or ...)"), + (yaml.DocumentEndToken, "document end markers (...)"), +) +# The frontmatter is limited further, to plain block mappings and scalars. +FRONTMATTER_FORBIDDEN_TOKENS = YAML_FORBIDDEN_TOKENS + ( + (yaml.DocumentStartToken, "document start markers (---)"), (yaml.FlowMappingStartToken, "flow collections ({...} or [...])"), (yaml.FlowSequenceStartToken, "flow collections ({...} or [...])"), ) @@ -393,6 +438,323 @@ def build_zip(files: dict[str, bytes]) -> bytes: return buffer.getvalue() +# --- YAML ------------------------------------------------------------------------- + +# A file line, or None when PyYAML gives no position, and a message. +YamlProblem = tuple[int | None, str] + + +def read_yaml( + text: str, subject: str, first_line: int, forbidden_tokens: tuple = YAML_FORBIDDEN_TOKENS +) -> tuple[Node | None, list[YamlProblem]]: + """The composed node tree of `text`, and each (line, message) problem that YAML parsers could read differently. + + The tree is None when there is a problem or no document. `subject` names the + text in the messages, such as "the frontmatter", and `first_line` is the + file line of the text's first line. + """ + problems = _yaml_text_problems(text, subject, first_line) or _yaml_token_problems( + text, subject, first_line, forbidden_tokens + ) + if problems: + return None, problems + try: + root = yaml.compose(text, Loader=yaml.SafeLoader) + except yaml.YAMLError as exc: + return None, [_yaml_error_problem(exc, subject, first_line)] + except RecursionError: + return None, [(first_line, f"{subject} is nested too deeply to read")] + if root is None: + return None, [] + problems = _yaml_node_problems(root, subject, first_line) + if problems: + return None, problems + return root, [] + + +def _yaml_text_problems(text: str, subject: str, first_line: int) -> list[YamlProblem]: + problems: list[YamlProblem] = [] + for pattern, problem in YAML_TEXT_PROBLEMS: + if match := pattern.search(text): + line = _line_at(text, match.start()) + first_line - 1 + problems.append((line, f"{subject} contains " + problem.format(code=ord(match.group())))) + return problems + + +def _yaml_token_problems( + text: str, subject: str, first_line: int, forbidden_tokens: tuple +) -> list[YamlProblem]: + problems: list[YamlProblem] = [] + open_flow_collections: list[yaml.Token] = [] + block_collection_columns: list[int] = [] + try: + for token in yaml.scan(text, Loader=yaml.SafeLoader): + line = token.start_mark.line + first_line + problems += [ + (line, f"{subject} uses {what}, which is not allowed") + for kind, what in forbidden_tokens + if isinstance(token, kind) + ] + # PyYAML reads a # right after a closing quote or bracket as a comment, and other parsers refuse it. + is_quoted_scalar = isinstance(token, yaml.ScalarToken) and token.style in ("'", '"') + is_flow_collection_end = isinstance(token, (yaml.FlowMappingEndToken, yaml.FlowSequenceEndToken)) + if (is_quoted_scalar or is_flow_collection_end) and text.startswith("#", token.end_mark.index): + message = ( + f"{subject} has a # right after a closing quote or bracket, which other YAML parsers don't read " + "as a comment; put a space before the #" + ) + problems.append((token.end_mark.line + first_line, message)) + if isinstance(token, (yaml.BlockMappingStartToken, yaml.BlockSequenceStartToken)): + block_collection_columns.append(token.start_mark.column) + elif isinstance(token, yaml.BlockEndToken) and block_collection_columns: + block_collection_columns.pop() + elif isinstance(token, (yaml.FlowMappingStartToken, yaml.FlowSequenceStartToken)): + open_flow_collections.append(token) + elif isinstance(token, (yaml.FlowMappingEndToken, yaml.FlowSequenceEndToken)) and open_flow_collections: + start = open_flow_collections.pop() + if not open_flow_collections and start.start_mark.line != token.start_mark.line: + problems.append( + ( + start.start_mark.line + first_line, + f"{subject} has a flow collection ({{...}} or [...]) over more than one line, which YAML " + "parsers read differently; keep it on one line or use block style", + ) + ) + elif isinstance(token, yaml.KeyToken) and open_flow_collections: + is_question_mark_before_text = text.startswith("?", token.start_mark.index) and bool( + text[token.end_mark.index:token.end_mark.index + 1].strip() + ) + if is_question_mark_before_text: + problems.append((line, f"{subject} {FLOW_PLAIN_SCALAR_PROBLEM}")) + # More open collections put the innermost past the depth limit; stopping here keeps deep input fast. + if len(open_flow_collections) + len(block_collection_columns) > MAX_YAML_DEPTH + 1: + problems.append((line, f"{subject} is nested too deeply; the limit is {MAX_YAML_DEPTH} levels")) + break + if isinstance(token, yaml.ScalarToken): + # Other parsers read a lone - in a flow collection as a block list item, unless a : follows it. + is_lone_dash_in_flow_collection = ( + token.plain + and token.value == "-" + and bool(open_flow_collections) + and not text.startswith(":", token.end_mark.index) + ) + if is_lone_dash_in_flow_collection: + problems.append((line, f"{subject} {FLOW_PLAIN_SCALAR_PROBLEM}")) + parent_indentation = block_collection_columns[-1] if block_collection_columns else -1 + problems += _scalar_source_problems(text, token, subject, first_line, parent_indentation) + except yaml.YAMLError as exc: + problems.append(_yaml_error_problem(exc, subject, first_line)) + except (ValueError, OverflowError): + # PyYAML raises these, not a YAMLError, for an escape past the last Unicode character. + problems.append((None, f"{subject} is not valid YAML: it has an escape past U+10FFFF")) + return problems + + +def _scalar_source_problems( + text: str, token: yaml.ScalarToken, subject: str, first_line: int, parent_indentation: int +) -> list[YamlProblem]: + """Problems in how a quoted or block scalar is written, which PyYAML reads without an error. + + `parent_indentation` is the column of the block mapping or list that holds the scalar, or -1 at the top. + """ + if token.style in ("'", '"'): + problems = _quoted_scalar_indentation_problems(text, token, subject, first_line, parent_indentation) + if token.style == '"': + problems += _double_quoted_escape_problems(text, token, subject, first_line) + return problems + if token.style in ("|", ">"): + return _block_scalar_problems(text, token, subject, first_line, parent_indentation) + return [] + + +def _quoted_scalar_indentation_problems( + text: str, token: yaml.ScalarToken, subject: str, first_line: int, parent_indentation: int +) -> list[YamlProblem]: + lines = text[token.start_mark.index:token.end_mark.index].split("\n") + for line_offset, line in enumerate(lines[1:], start=1): + indentation = len(line) - len(line.lstrip(" ")) + # PyYAML takes a tab, or too few spaces, as the indentation of a quoted value's next line; others refuse it. + if line.strip(" \r") and indentation <= parent_indentation: + line_number = token.start_mark.line + line_offset + first_line + message = ( + f"{subject} has a quoted value whose next line isn't indented past its key or list item with spaces, " + "which YAML parsers read differently; indent the line with more spaces" + ) + return [(line_number, message)] + return [] + + +def _double_quoted_escape_problems( + text: str, token: yaml.ScalarToken, subject: str, first_line: int +) -> list[YamlProblem]: + start = token.start_mark.index + problems: list[YamlProblem] = [] + for escape in DOUBLE_QUOTED_ESCAPE_PATTERN.finditer(text[start:token.end_mark.index]): + utf16_hex, utf32_hex, line_break_before_blank_line = escape.groups() + hex_digits = utf16_hex or utf32_hex + line = _line_at(text, start + escape.start()) + first_line - 1 + # PyYAML decodes each escape on its own, so even an escaped surrogate pair stays two surrogates. + if hex_digits and 0xD800 <= int(hex_digits, 16) <= 0xDFFF: + problems.append( + ( + line, + f"{subject} has an escape for a UTF-16 surrogate (U+D800 to U+DFFF), which can't be written " + "as UTF-8; write the character itself", + ) + ) + elif line_break_before_blank_line is not None: + problems.append( + ( + line, + f"{subject} has an escaped line break before a blank line in a double-quoted value, which " + "YAML parsers read differently", + ) + ) + return problems + + +def _block_scalar_problems( + text: str, token: yaml.ScalarToken, subject: str, first_line: int, parent_indentation: int +) -> list[YamlProblem]: + start, end = token.start_mark.index, token.end_mark.index + header, line_break, content = text[start:end].partition("\n") + indicators = BLOCK_SCALAR_INDICATORS_PATTERN.match(header).group() + line = token.start_mark.line + first_line + if any(character.isdigit() for character in indicators): + message = ( + f"{subject} has a block scalar indentation indicator, which YAML parsers read differently; " + "remove the digit after | or >" + ) + return [(line, message)] + problems: list[YamlProblem] = [] + leading_lines = BLOCK_SCALAR_LEADING_LINES_PATTERN.match(text, start + len(header) + 1) if line_break else None + first_line_indentation = len(leading_lines.group(2)) if leading_lines else None + if leading_lines: + longest_blank_line = max(len(blank_line) for blank_line in leading_lines.group(1).replace("\r", "").split("\n")) + # PyYAML takes the content's indentation from the longest blank line before it, others from the first line. + if parent_indentation < first_line_indentation < longest_blank_line: + message = ( + f"{subject} has a block scalar whose first line is indented less than a blank line before it, which " + "YAML parsers read differently; remove the spaces from the blank line" + ) + problems.append((line, message)) + is_strip_chomped = "-" in indicators + is_keep_chomped = "+" in indicators + is_missing_final_line_break = bool(content) and not content.endswith("\n") and not text.endswith("\n") + # Other parsers keep a final line of spaces past the parent's indentation in the scalar when the scalar has no + # lines before it, or when the spaces reach past the content's indentation, or reach it for |+ and >+. + final_spaces = len(text) - end if not text[end:].strip(" ") else 0 + if final_spaces and first_line_indentation is None: + keeps_final_spaces = final_spaces > parent_indentation and (is_keep_chomped or content == "") + elif final_spaces: + keeps_final_spaces = final_spaces > first_line_indentation or ( + is_keep_chomped and final_spaces == first_line_indentation + ) + else: + keeps_final_spaces = False + # PyYAML leaves out the final line break that other parsers add, unless |- or >- strips it for both. + if not is_strip_chomped and (is_missing_final_line_break or keeps_final_spaces): + message = ( + f"{subject} ends in a block scalar with no line break after it, which YAML parsers read " + "differently; use |- or >-, or put another key after it" + ) + problems.append((line, message)) + return problems + + +def _yaml_error_problem(exc: yaml.YAMLError, subject: str, first_line: int) -> YamlProblem: + mark = getattr(exc, "problem_mark", None) + problem = getattr(exc, "problem", None) or str(exc) + line = mark.line + first_line if mark else None + return line, f"{subject} is not valid YAML: {problem}" + + +def _yaml_node_problems(root: Node, subject: str, first_line: int) -> list[YamlProblem]: + """Problems at any depth: nesting, keys that aren't plain text or that repeat, and unquoted values. + + PyYAML keeps the last value of a repeated key, while other parsers reject the file. + """ + problems: list[YamlProblem] = [] + # Each entry is a node, the key path to it, its depth, and whether it is inside a flow collection. + stack: list[tuple[Node, str, int, bool]] = [(root, "", 0, False)] + while stack: + node, prefix, depth, is_in_flow_collection = stack.pop() + line = node.start_mark.line + first_line + if depth > MAX_YAML_DEPTH: + problems.append((line, f"{subject} is nested too deeply; the limit is {MAX_YAML_DEPTH} levels")) + elif isinstance(node, ScalarNode) and node.style is None: + problem = _plain_scalar_problem(node.value, is_in_flow_collection, is_key=False) + if problem: + problems.append((line, f"{subject} {problem}")) + elif isinstance(node, SequenceNode): + is_item_in_flow_collection = is_in_flow_collection or bool(node.flow_style) + stack += [(item, prefix, depth + 1, is_item_in_flow_collection) for item in node.value] + elif isinstance(node, MappingNode): + is_item_in_flow_collection = is_in_flow_collection or bool(node.flow_style) + seen: set[str] = set() + for key_node, value_node in node.value: + key_line = key_node.start_mark.line + first_line + key_problem = _key_problem(key_node, is_item_in_flow_collection) + if key_problem: + problems.append((key_line, f"{subject} {key_problem}")) + continue + if key_node.value in seen: + problems.append((key_line, f'{subject} has the duplicate key "{prefix}{key_node.value}"')) + seen.add(key_node.value) + stack.append((value_node, f"{prefix}{key_node.value}.", depth + 1, is_item_in_flow_collection)) + return sorted(problems) + + +def _key_problem(key_node: Node, is_in_flow_collection: bool) -> str | None: + if not isinstance(key_node, ScalarNode): + return "keys must be plain text, not a list or mapping" + if key_node.value == "<<": + return "has a << key, which YAML 1.1 parsers read as a merge; rename it" + if key_node.style is None: + return _plain_scalar_problem(key_node.value, is_in_flow_collection, is_key=True) + return None + + +def _plain_scalar_problem(text: str, is_in_flow_collection: bool, *, is_key: bool) -> str | None: + """Why YAML parsers could read an unquoted key or value differently, or None.""" + if is_in_flow_collection and FLOW_PLAIN_SCALAR_PATTERN.search(text): + return FLOW_PLAIN_SCALAR_PROBLEM + if not _is_read_as_non_text_by_some_parser(text): + return None + if is_key and text == "": + return "keys must be plain text, not empty" + if is_key: + return ( + f"keys must be plain text, but YAML parsers don't all read the unquoted key {text} as text; " + f'put it in quotes, as in "{text}"' + ) + if _is_typed_alike_by_every_parser(text): + return None + return ( + f"has the unquoted value {text}, which YAML parsers don't all read as the same type; " + f'put it in quotes, as in "{text}"' + ) + + +def _is_read_as_non_text_by_some_parser(text: str) -> bool: + return any(pattern.fullmatch(text) for pattern in NON_TEXT_PLAIN_SCALAR_PATTERNS) + + +def _is_typed_alike_by_every_parser(text: str) -> bool: + """True for true, false, null, ~, an empty value, and an integer or decimal every parser reads as the same number.""" + if text in ("true", "false", "null", "~", ""): + return True + if CANONICAL_INTEGER_PATTERN.fullmatch(text): + digits = text.removeprefix("-") + # int() refuses more than 4,300 digits, and no number that long is safe anyway. + return text != "-0" and len(digits) <= len(str(MAX_SAFE_INTEGER)) and int(digits) <= MAX_SAFE_INTEGER + if CANONICAL_DECIMAL_PATTERN.fullmatch(text): + decimal = float(text) + # repr gives back the text only for a decimal written the shortest way, so 1.10 stays text. + return repr(decimal) == text and not decimal.is_integer() and abs(decimal) >= MIN_DECIMAL_MAGNITUDE + return False + + # --- SKILL.md ----------------------------------------------------------------- @@ -410,7 +772,7 @@ def warning(self, message: str, line: int | None = None) -> None: def _line(node: Node) -> int: - return node.start_mark.line + 2 + return node.start_mark.line + FRONTMATTER_FIRST_LINE def _kind(node: Node) -> str: @@ -449,13 +811,6 @@ def _check_skill_md(folder: str, data: bytes, rules: Rules, findings: list[Findi line = frontmatter.count("\n", 0, frontmatter.index("\t")) + 2 report.error("the frontmatter contains a tab character; use spaces", line=line) return None - if line_break := YAML11_LINE_BREAK_PATTERN.search(frontmatter): - report.error( - f"the frontmatter contains U+{ord(line_break.group()):04X}, which YAML parsers disagree on as a line " - "break; use a newline", - line=frontmatter.count("\n", 0, line_break.start()) + 2, - ) - return None root = _parse_frontmatter(frontmatter, report) if root is None: @@ -464,7 +819,7 @@ def _check_skill_md(folder: str, data: bytes, rules: Rules, findings: list[Findi report.error("the frontmatter must be a mapping of keys to values, such as name: my-skill", line=2) return None - top = _mapping_items(root, "", report) + top = _mapping_items(root) for key, (key_node, _) in top.items(): if key not in TOP_LEVEL_KEYS: close = difflib.get_close_matches(key, TOP_LEVEL_KEYS, n=1) @@ -504,93 +859,28 @@ def _parse_frontmatter(frontmatter: str, report: _Report, *, strict: bool = True return yaml.compose(frontmatter, Loader=yaml.SafeLoader) except (yaml.YAMLError, RecursionError): return None - try: - forbidden = [ - (token, what) - for token in yaml.scan(frontmatter, Loader=yaml.SafeLoader) - for kind, what in _FORBIDDEN_TOKENS - if isinstance(token, kind) - ] - except yaml.YAMLError as exc: - _report_yaml_error(exc, report) - return None - for token, what in forbidden: - report.error(f"the frontmatter uses {what}, which is not allowed", line=token.start_mark.line + 2) - if forbidden: + root, problems = read_yaml(frontmatter, "the frontmatter", FRONTMATTER_FIRST_LINE, FRONTMATTER_FORBIDDEN_TOKENS) + for line, message in problems: + report.error(message, line=line) + if problems: return None marker = frontmatter.find("---") if marker != -1: # Some readers end the frontmatter at the first "---", even inside a value. report.error( 'the frontmatter contains "---", which other tools read as its end', - line=frontmatter.count("\n", 0, marker) + 2, + line=frontmatter.count("\n", 0, marker) + FRONTMATTER_FIRST_LINE, ) return None - try: - root = yaml.compose(frontmatter, Loader=yaml.SafeLoader) - except yaml.YAMLError as exc: - _report_yaml_error(exc, report) - return None - except RecursionError: - report.error("the frontmatter is nested too deeply to read", line=2) - return None if root is None: - report.error("the frontmatter is empty", line=2) - return None - if _report_key_problems(root, report): + report.error("the frontmatter is empty", line=FRONTMATTER_FIRST_LINE) return None return root -def _report_key_problems(root: Node, report: _Report) -> bool: - """Report keys, in any mapping at any depth, that parsers could read differently. True if there were any. - - A key must be plain text that no parser reads as a number, true/false or - null, and must not repeat: PyYAML keeps the last value of a repeated key, - while other parsers reject the file. - """ - problems: list[tuple[int, str]] = [] - stack: list[tuple[Node, str]] = [(root, "")] - while stack: - node, prefix = stack.pop() - if isinstance(node, MappingNode): - seen: set[str] = set() - for key_node, value_node in node.value: - if not ( - isinstance(key_node, ScalarNode) - and key_node.tag == STR_TAG - and not (key_node.style is None and YAML12_NUMBER_PATTERN.fullmatch(key_node.value)) - ): - problems.append((_line(key_node), "frontmatter keys must be plain text")) - continue - if key_node.value in seen: - problems.append((_line(key_node), f'duplicate key "{prefix}{key_node.value}"')) - seen.add(key_node.value) - stack.append((value_node, f"{prefix}{key_node.value}.")) - elif isinstance(node, SequenceNode): - stack += [(item, prefix) for item in node.value] - for line, message in sorted(problems): - report.error(message, line=line) - return bool(problems) - - -def _report_yaml_error(exc: yaml.YAMLError, report: _Report) -> None: - mark = getattr(exc, "problem_mark", None) - problem = getattr(exc, "problem", None) or str(exc) - report.error(f"the frontmatter is not valid YAML: {problem}", line=mark.line + 2 if mark else None) - - -def _mapping_items(node: MappingNode, prefix: str, report: _Report) -> dict[str, tuple[ScalarNode, Node]]: - items: dict[str, tuple[ScalarNode, Node]] = {} - for key_node, value_node in node.value: - if not (isinstance(key_node, ScalarNode) and key_node.tag == STR_TAG): - report.error("frontmatter keys must be plain text", line=_line(key_node)) - continue - if key_node.value in items: - report.error(f'duplicate key "{prefix}{key_node.value}"', line=_line(key_node)) - continue - items[key_node.value] = (key_node, value_node) - return items +def _mapping_items(node: MappingNode) -> dict[str, tuple[ScalarNode, Node]]: + """The key and value nodes by key. read_yaml has already refused keys that aren't text or that repeat.""" + return {key_node.value: (key_node, value_node) for key_node, value_node in node.value} def _text(items: dict, key: str, label: str, report: _Report, *, required: bool = False) -> str | None: @@ -599,13 +889,6 @@ def _text(items: dict, key: str, label: str, report: _Report, *, required: bool return None key_node, value_node = items[key] if isinstance(value_node, ScalarNode) and value_node.tag == STR_TAG: - if value_node.style is None and YAML12_NUMBER_PATTERN.fullmatch(value_node.value): - report.error( - f'"{label}" must be text, but other YAML parsers read {value_node.value} as a number. ' - f'Put the value in quotes, for example {key}: "{value_node.value}"', - line=_line(key_node), - ) - return None return value_node.value if isinstance(value_node, ScalarNode) and value_node.tag == "tag:yaml.org,2002:null": advice = "give it a value" if required else "give it a value or remove the key" @@ -713,7 +996,7 @@ def _check_metadata( report.error('"metadata" must be a mapping, with one key: value pair per line', line=metadata_line) return {}, None, {} - items = _mapping_items(node, "metadata.", report) + items = _mapping_items(node) metadata: dict[str, str] = {} lines: dict[str, int] = {} for key in items: diff --git a/.github/scripts/validate_skills.py b/.github/scripts/validate_skills.py index 97ab9db4c..b5c44ad65 100644 --- a/.github/scripts/validate_skills.py +++ b/.github/scripts/validate_skills.py @@ -43,6 +43,9 @@ SKILLS_DIR = "skills" CONNECTORS_DIR = "connectors" +CONNECTOR_FILE = "connector.yaml" +MAX_CONNECTOR_FILE_BYTES = 32_768 +RELEASES_PATH = f"{CONNECTORS_DIR}/releases.yaml" RULES_DIR = Path(__file__).resolve().parent / "skill-rules" CONFORMANCE_FILE = "conformance-cases.json" DOCS_HINT = 'See the "Skill Publishing Rules" section of CONTRIBUTING.md.' @@ -246,11 +249,11 @@ def retired_skills(repo_root: Path, base: str, present: set[str]) -> frozenset[s def connector_skill_references(repo_root: Path, rev: str) -> tuple[dict[str, list[str]], list[SkillResult]]: """Skill name -> the connectors whose connector.yaml lists it under skills: (recommended skills). - Also returns a warning for each connector.yaml that can't be read, isn't a - mapping, has a skills: value that isn't a list of skill names, or writes - skills: more than once, so a broken file doesn't silently skip the check. The YAML is composed, not - constructed, so a value PyYAML can't build (a bad date, say) can't stop - the check. + Also returns an error for each connector.yaml, and for connectors/releases.yaml, + that isn't UTF-8 or that breaks the YAML rules of sc.read_yaml, since the + connectors are published from the same files. A connector.yaml that isn't a + mapping or has a skills: value that isn't a list of skill names gets a + warning, so a broken file doesn't silently skip the check. """ listing = _git(repo_root, "ls-tree", "-r", "-z", "--full-tree", rev, "--", f"{CONNECTORS_DIR}/") files: list[tuple[str, str]] = [] @@ -259,58 +262,53 @@ def connector_skill_references(repo_root: Path, rev: str) -> tuple[dict[str, lis continue meta, _, raw_path = record.partition(b"\t") _, kind, oid = meta.decode("ascii").split(" ") - parts = _decode_path(raw_path).split("/") - if kind == "blob" and len(parts) == 3 and parts[2] == "connector.yaml": - files.append((parts[1], oid)) + path = _decode_path(raw_path) + parts = path.split("/") + is_connector_file = len(parts) == 3 and parts[2] == CONNECTOR_FILE + if kind == "blob" and (is_connector_file or path == RELEASES_PATH): + files.append((path, oid)) references: dict[str, list[str]] = {} - unreadable: list[SkillResult] = [] - for (connector, _), data in zip(files, _read_blobs(repo_root, [oid for _, oid in files])): - try: - problem, skills = _recommended_skills(*_compose_connector(data.decode("utf-8"))) - except (UnicodeDecodeError, yaml.YAMLError, RecursionError): - problem, skills = "can't be read as YAML, so its skills: list wasn't checked", [] - if problem: - message = f"{problem} against deprecated skills" - path = f"{CONNECTORS_DIR}/{connector}/connector.yaml" - unreadable.append(SkillResult(path, [sc.Finding("warning", message)], is_skill=False, always_visible=True)) - for skill in skills: - references.setdefault(skill, []).append(connector) - return references, unreadable - - -def _compose_connector(text: str) -> tuple[yaml.Node | None, int]: - """The YAML node tree of a connector.yaml, with merge keys (<<) resolved, and how many skills: keys it writes. - - No values are built. The count is taken before merging, since a merge can - supply skills: as well without the file being ambiguous. - """ - loader = yaml.SafeLoader(text) + results: list[SkillResult] = [] + for (path, _), data in zip(files, _read_blobs(repo_root, [oid for _, oid in files])): + root, findings = _read_connector_yaml(path, data) + if path != RELEASES_PATH and not findings: + problem, skills = _recommended_skills(root) + if problem: + findings.append(sc.Finding("warning", f"{problem} against deprecated skills")) + for skill in skills: + references.setdefault(skill, []).append(path.split("/")[1]) + if findings: + results.append(SkillResult(path, findings, is_skill=False, always_visible=True)) + return references, results + + +def _read_connector_yaml(path: str, data: bytes) -> tuple[yaml.Node | None, list[sc.Finding]]: + """The YAML node tree of a file under connectors/, or None with an error for each problem.""" + file_name = path.rsplit("/", 1)[-1] + if file_name == CONNECTOR_FILE and len(data) > MAX_CONNECTOR_FILE_BYTES: + message = f"{file_name} is {len(data):,} bytes; the limit is {MAX_CONNECTOR_FILE_BYTES:,}" + return None, [sc.Finding("error", message)] try: - root = loader.get_single_node() - written = 0 - if isinstance(root, yaml.MappingNode): - written = sum(1 for key, _ in root.value if isinstance(key, yaml.ScalarNode) and key.value == "skills") - loader.flatten_mapping(root) - return root, written - finally: - loader.dispose() + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + return None, [sc.Finding("error", f"{file_name} is not valid UTF-8", line=data.count(b"\n", 0, exc.start) + 1)] + root, problems = sc.read_yaml(text, file_name, first_line=1) + return root, [sc.Finding("error", message, line=line) for line, message in problems] -def _recommended_skills(root: yaml.Node | None, written: int) -> tuple[str | None, list[str]]: +def _recommended_skills(root: yaml.Node | None) -> tuple[str | None, list[str]]: """The skill names under a connector.yaml's skills: key, and what kept any of it from being checked.""" if not isinstance(root, yaml.MappingNode): return "isn't a YAML mapping, so its skills: list wasn't checked", [] - values = [value for key, value in root.value if isinstance(key, yaml.ScalarNode) and key.value == "skills"] - if not values: + skills_node = next((value for key, value in root.value if key.value == "skills"), None) + if skills_node is None: return None, [] - if not isinstance(values[-1], yaml.SequenceNode): + if not isinstance(skills_node, yaml.SequenceNode): return "has a skills: value that isn't a list, so it wasn't checked", [] - items = values[-1].value # As in PyYAML, the last skills: key wins. + items = skills_node.value names = [item.value for item in items if isinstance(item, yaml.ScalarNode) and item.tag == sc.STR_TAG] if len(names) < len(items): return "has a skills: entry that isn't a skill name, so that entry wasn't checked", names - if written > 1: - return "has more than one skills: key, so only the last one was checked", names return None, names @@ -342,8 +340,8 @@ def check_repository( history[name] = sc.check_history(name, base_trees.get(name), head_trees.get(name), rules, retired) for name in sorted(set(base_trees) - set(head_trees)): results.append(SkillResult(f"{SKILLS_DIR}/{name}", history[name], is_skill=False)) - recommended, unreadable = connector_skill_references(repo_root, head_ref) - results += unreadable + recommended, connector_results = connector_skill_references(repo_root, head_ref) + results += connector_results for name in sorted(head_trees): findings, info = sc.check_skill(head_trees[name], rules) if base is not None: diff --git a/.github/workflows/validate-skills-main.yml b/.github/workflows/validate-skills-main.yml index 109d9e21d..6b0fe27b6 100644 --- a/.github/workflows/validate-skills-main.yml +++ b/.github/workflows/validate-skills-main.yml @@ -24,6 +24,7 @@ jobs: validate-skills-main: name: validate-skills-main runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: diff --git a/.github/workflows/validate-skills.yml b/.github/workflows/validate-skills.yml index a4c4a10de..6491b1eff 100644 --- a/.github/workflows/validate-skills.yml +++ b/.github/workflows/validate-skills.yml @@ -46,6 +46,7 @@ jobs: validate-skills: name: validate-skills runs-on: ubuntu-latest + timeout-minutes: 10 steps: # Actions are pinned to full commit SHAs (v4.4.0 and v5.6.0) so a moved tag # can't change what runs here. diff --git a/.kiro/steering/project-conventions.md b/.kiro/steering/project-conventions.md index 5f792c97c..078cb069d 100644 --- a/.kiro/steering/project-conventions.md +++ b/.kiro/steering/project-conventions.md @@ -92,7 +92,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar `.github/workflows/validate-skills.yml` runs `.github/scripts/validate_skills.py` on every pull request, and `.github/workflows/validate-skills-main.yml` runs it again after each push to `main`. `.github/workflows/build-docs.yml` runs `mkdocs build --strict` on pull requests and on `main`. The rules and their reasons are in the "Skill Publishing Rules" section of [CONTRIBUTING.md](../../CONTRIBUTING.md). -- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, and a deprecated skill still listed in a `connectors/*/connector.yaml`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. +- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. - Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. @@ -101,7 +101,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. - `check_added_content` warns about internal host or tool names only on lines a PR adds, like `scan-aws-identifiers`, so names already on `main` don't warn on every PR. - With `--base-ref`, `check_history` compares each skill with its merge-base copy: no removed folders, no reuse of a path deleted in `main`'s first-parent history (`git log --first-parent --no-renames --diff-filter=D`, skipped with a warning on a shallow clone), a higher version whenever `content_hash` of the published files changes, one step at a time (next patch, minor or major), and `1.0.0` for a new skill. No folder is ever deleted, not even in an emergency: deprecate it instead. The hash function is pinned by the golden fixture under `content_hash` in `conformance-cases.json`, so any other implementation can prove it computes the same hash. -- With or without a base, a deprecated skill can't stay in any `connectors/*/connector.yaml` `skills:` list (`connector_skill_references`, which composes the YAML rather than constructing it). A `connector.yaml` that can't be read, isn't a mapping, has a `skills:` value that isn't a list of skill names, or writes `skills:` more than once gets a warning; the names it does list are still checked. +- With or without a base, a deprecated skill can't stay in any `connectors/*/connector.yaml` `skills:` list (`connector_skill_references`). Every `connectors/*/connector.yaml` and `connectors/releases.yaml` is read with `skill_checks.read_yaml`, the reader the frontmatter uses, so a file that isn't UTF-8 or breaks a YAML rule is an error; the cases are in `skill-rules/yaml-conformance-cases.json`, which `tests/test_skill_checks.py` runs. A `connector.yaml` that isn't a mapping or has a `skills:` value that isn't a list of skill names gets a warning; the names it does list are still checked. - Secrets in test fixtures are built at runtime (`"AKIA" + ...`) or stored as base64 in the conformance cases, so no file in the repository contains a literal key or invisible character. Internal names in tests are made up (`corp.example`, `Rivendell`), never real hosts or tools. - `conformance-cases.json` holds language-neutral cases for these rules. Another tool that validates or publishes these skills should give the same `expect` result on the cases for the rules it implements; some rules here are stricter than any other tool's. The script runs every case before checking any skill and exits 2 on a mismatch. A rule change needs a matching case change in the same PR. - Every rule applies to every skill; nothing is exempted. A rule that existing skills don't meet yet starts as a warning and becomes an error after a cleanup PR brings every skill into line: a missing `metadata.summary`, and dimension values outside the vocabulary. A missing `metadata.aws-devops-agent-skills.agent-types` stays a warning. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed15ad513..67c004faf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,7 +72,7 @@ Skills on `main` are published automatically, so a pull request check ([`.github | `metadata.aws-devops-agent-skills.agent-types` | The agents that load the skill, as comma-separated values from [`agent-types.json`](.github/scripts/skill-rules/agent-types.json), such as `"INCIDENT_RCA"`. An unknown value is an error, because this key decides which agents load the skill rather than only how catalogs filter it. `GENERIC` loads the skill for every agent, so it is an error to combine it with another value. Without the key the skill installs as `GENERIC` and loads for every agent, which is a warning. | | `metadata.aws-devops-agent-skills.*` | Only `agent-types`, `aws-services` and `technical-domains`, each a comma-separated list. Any other key under this prefix, or a misspelling of the prefix, is an error, so a typo can't drop the skill from a filter. For `aws-services` and `technical-domains`, a value not in [`vocabulary.json`](.github/scripts/skill-rules/vocabulary.json) is a warning, with the approved spelling when there is one. | -The frontmatter must be plain YAML that every parser reads the same way. These are rejected: tabs; a byte order mark at the start of the file; line breaks other than newlines (U+0085, U+2028, U+2029); `---` anywhere but the closing line; anchors (`&`), aliases (`*`), tags (`!`), directives (`%`) and document markers; flow collections (`[...]` or `{...}`); keys that aren't plain text, and duplicate keys, at any depth; and unquoted values that some parsers read as numbers, such as `1e3`. A top-level key other than the Agent Skills specification's fields and `title` is a warning. +The frontmatter must be plain YAML that every parser reads the same way. These are rejected: tabs; a byte order mark anywhere; line breaks other than newlines (U+0085, U+2028, U+2029, or a carriage return without a line feed); `---` anywhere but the closing line; anchors (`&`), aliases (`*`), tags (`!`), directives (`%`) and document markers; flow collections (`[...]` or `{...}`); keys that aren't plain text, `<<` keys, and duplicate keys, at any depth; nesting more than 64 levels deep; unquoted values that some parsers read as another type, such as `1e3`, `yes`, `y` or `2026-11-08`; in a double-quoted value, an escaped UTF-16 surrogate such as `\uD83D`, or an escaped line break before a blank line; a block scalar indentation indicator, such as `|2`; a block scalar whose first line is indented less than a blank line before it; a `#` right after a closing quote or bracket; a quoted value whose next line isn't indented past its key or list item with spaces; and a block scalar as the last key, unless it is `|-` or `>-`. A top-level key other than the Agent Skills specification's fields and `title` is a warning. A `SKILL.md` body over 500 lines is a warning, as it is in the skill evaluation tool: move reference material into `references/` and link to it. @@ -106,7 +106,7 @@ Identity and versions, compared with the base branch: - A new skill starts at `1.0.0`. - When the version goes up, add a `CHANGELOG.md` entry for it. A bump without one is a warning. -Connectors: a skill that a connector recommends, in the `skills:` list of a `connectors//connector.yaml`, can't be deprecated unless the same pull request removes it from that list. A `connector.yaml` that can't be read is a warning. +Connectors: a skill that a connector recommends, in the `skills:` list of a `connectors//connector.yaml`, can't be deprecated unless the same pull request removes it from that list. Each `connector.yaml`, at most 32 KiB, and `connectors/releases.yaml` must be UTF-8 YAML that every parser reads the same way, by the frontmatter's rules, except that a flow collection on one line, a `---` line at the start, `---` inside a value, and a tab inside a quoted value, block scalar or comment are allowed. A flow collection can't span lines, and an unquoted entry in one can't start with `?` or `:`, hold a `?`, or be a lone `-` that no `:` follows. [`yaml-conformance-cases.json`](.github/scripts/skill-rules/yaml-conformance-cases.json) has a case for each rule. A `connector.yaml` that isn't a mapping, or whose `skills:` isn't a list of skill names, is a warning. The check is exact only for the base commit it ran on. Two pull requests that each pass on an older base can break the rules together, for example by both bumping a skill to the same version, so a later push to `main` can still fail. [`validate-skills-main.yml`](.github/workflows/validate-skills-main.yml) runs the check again after every push to `main` and reports such a break. @@ -124,7 +124,7 @@ git fetch upstream main python3 .github/scripts/validate_skills.py --base-ref upstream/main ``` -Before it checks any skill, the script runs the cases in [`conformance-cases.json`](.github/scripts/skill-rules/conformance-cases.json) through the same rules. When you change a rule, add or update a case for it in the same pull request. `python3 .github/scripts/validate_skills.py --self-check-only` runs only the cases, and `python3 -m unittest discover -s tests -v` runs the check's unit tests. +Before it checks any skill, the script runs the cases in [`conformance-cases.json`](.github/scripts/skill-rules/conformance-cases.json) through the same rules. When you change a rule, add or update a case for it in the same pull request; the YAML rules also have cases in [`yaml-conformance-cases.json`](.github/scripts/skill-rules/yaml-conformance-cases.json), which the unit tests run. `python3 .github/scripts/validate_skills.py --self-check-only` runs only the cases, and `python3 -m unittest discover -s tests -v` runs the check's unit tests. #### Test Your Skill diff --git a/tests/test_skill_checks.py b/tests/test_skill_checks.py index 897108728..ad07ba742 100644 --- a/tests/test_skill_checks.py +++ b/tests/test_skill_checks.py @@ -19,6 +19,8 @@ import skill_checks as sc # noqa: E402 +YAML_CASES_FILE = ROOT / ".github" / "scripts" / "skill-rules" / "yaml-conformance-cases.json" + RULES = sc.Rules( agent_types=frozenset({"GENERIC", "CHAT", "INCIDENT_RCA"}), vocabulary={ @@ -211,8 +213,8 @@ def test_triple_dash_inside_the_frontmatter_is_rejected(self): self.assertOneError(tree(skill_md(fm)), 'contains "---"', line=3) def test_directives_and_document_markers_are_rejected(self): - self.assertOneError(tree(b"---\n%YAML 1.1\n--- # start\n" + skill_md()[4:]), "directives (%)", also=("document markers",)) - self.assertOneError(tree(skill_md(VALID_FRONTMATTER + "...\n")), "document markers", line=9) + self.assertOneError(tree(b"---\n%YAML 1.1\n--- # start\n" + skill_md()[4:]), "directives (%)", also=("document start markers",)) + self.assertOneError(tree(skill_md(VALID_FRONTMATTER + "...\n")), "document end markers", line=9) def test_empty_frontmatter_is_named(self): for content in (b"---\n\n---\n# Demo\n", b"---\n---\n# Demo\n"): @@ -220,6 +222,97 @@ def test_empty_frontmatter_is_named(self): self.assertOneError(tree(content), "the frontmatter is empty") +class ReadYamlTests(unittest.TestCase): + def messages(self, text: str) -> list[str]: + return [message for _, message in sc.read_yaml(text, "connector.yaml", first_line=1)[1]] + + def test_shared_yaml_cases_give_their_expected_results(self): + cases = json.loads(YAML_CASES_FILE.read_text(encoding="utf-8"))["cases"] + self.assertTrue(cases) + ids = [case["id"] for case in cases] + self.assertEqual(len(ids), len(set(ids))) + for case in cases: + with self.subTest(case["id"]): + self.assertIn(case["expect"], ("pass", "fail")) + if case["expect"] == "fail": + self.assertIsInstance(case.get("error"), str) + self.assertTrue(case["error"]) + messages = self.messages(case["yaml"]) + if case["expect"] == "pass": + self.assertEqual(messages, []) + else: + self.assertTrue(any(case["error"] in message for message in messages), messages) + + def test_problems_are_reported_on_file_lines(self): + root, problems = sc.read_yaml("a: 1\nb: !!str x\n", "the frontmatter", first_line=2) + self.assertIsNone(root) + self.assertEqual(problems, [(3, "the frontmatter uses tags (! or !!), which is not allowed")]) + for text, line in (('a: "x\n \\uD800"\n', 2), ("a: 1\nb: [x,\n y]\n", 2), ("a:\n b: 1\n b: 2\n", 3)): + with self.subTest(text=text): + self.assertEqual([line for line, _ in sc.read_yaml(text, "connector.yaml", first_line=1)[1]], [line]) + + def test_numbers_are_typed_alike_only_within_the_safe_range(self): + for value in ("9007199254740991", "-9007199254740991", "0.0001", "-1.5", "123456789012345.6"): + with self.subTest(accepts=value): + self.assertEqual(self.messages(f"a: {value}\n"), []) + for value in ("9007199254740992", "0.00009", "2.0", "-0.0", "1" * 5000): + with self.subTest(rejects=value[:20]): + (message,) = self.messages(f"a: {value}\n") + self.assertIn("don't all read as the same type", message) + + def test_numbers_only_yaml_12_parsers_read_are_rejected_too(self): + for value in ("1_0e3", "0o1_0", "._5"): + with self.subTest(value=value): + self.assertIn("the same type", self.messages(f"a: {value}\n")[0]) + self.assertIn("keys must be plain text", self.messages(f"{value}: a\n")[0]) + + def test_a_lone_dash_in_a_flow_collection_is_rejected_unless_a_colon_follows_it(self): + for text in ("k: [a, -]\n", "k: {a: -}\n", "k: {-}\n"): + with self.subTest(text=text): + self.assertIn("is a lone -", self.messages(text)[0]) + for text in ("k: {-: v}\n", "k: [-: v]\n", "k: [-a]\n"): + with self.subTest(text=text): + self.assertEqual(self.messages(text), []) + + def test_a_quoted_value_line_is_indented_past_the_collection_that_holds_it(self): + rejected = {"k:\n j: \"a\n b\"\n": 3, "k:\n - 'a\n b'\n": 3, "k:\n- \"a\nb\"\n": 3, "k: \"a\n\t\n b\"\n": 2} + for text, line in rejected.items(): + with self.subTest(text=text): + ((problem_line, message),) = sc.read_yaml(text, "connector.yaml", first_line=1)[1] + self.assertEqual(problem_line, line) + self.assertIn("isn't indented past its key or list item", message) + for text in ("k:\n j: \"a\n b\"\n", "k:\n- \"a\n b\"\n", "\"a\nb\"\n", "k: \"a\n\n b\"\n", "k: \"a\n \t\n b\"\n"): + with self.subTest(text=text): + self.assertEqual(self.messages(text), []) + + def test_block_scalar_lines_are_measured_from_the_collection_that_holds_it(self): + self.assertIn("less than a blank line", self.messages("a:\n b: |\n \n # c\n c: 1\n")[0]) + self.assertEqual(self.messages("a:\n b: |\n \n # c\n c: 1\n"), []) + self.assertEqual(self.messages("a:\n - >+\n\n "), []) + self.assertIn("no line break after it", self.messages("a: >+\n\n ")[0]) + for text in ("a: |\n x\n ", "a: |+\n x\n "): + with self.subTest(text=text): + self.assertIn("no line break after it", self.messages(text)[0]) + for text in ("a: |\n x\n ", "a: |+\n x\n ", "a: |-\n x\n ", "a: |\n\n "): + with self.subTest(text=text): + self.assertEqual(self.messages(text), []) + + def test_deep_nesting_stops_the_scan_at_the_limit(self): + for text in ("a: " + "[" * 20000, "a: " + "{" * 20000): + with self.subTest(text=text[:5]): + self.assertEqual(self.messages(text), ["connector.yaml is nested too deeply; the limit is 64 levels"]) + self.assertEqual(self.messages("a: " + "[" * 63 + "x" + "]" * 63 + "\n"), []) + self.assertIn("nested too deeply", self.messages("a: " + "[" * 64 + "x" + "]" * 64 + "\n")[0]) + + def test_an_empty_key_is_named(self): + self.assertEqual(self.messages("? \n: v\n"), ["connector.yaml keys must be plain text, not empty"]) + + def test_an_escape_past_the_last_unicode_character_is_a_problem_not_a_crash(self): + for escape in ("\\U00110000", "\\UFFFFFFFF"): + with self.subTest(escape=escape): + self.assertIn("escape past U+10FFFF", self.messages(f'a: "{escape}"\n')[0]) + + class NameTests(CheckSkillTestCase): def with_name(self, name: str, folder: str = "demo-skill") -> sc.SkillTree: return tree(skill_md(VALID_FRONTMATTER.replace("name: demo-skill", f"name: {name}")), name=folder) @@ -274,7 +367,7 @@ def test_description_limit(self): def test_title_limit_and_single_line(self): fm = VALID_FRONTMATTER + f"title: {'T' * 101}\n" self.assertOneError(tree(skill_md(fm)), '"title" is 101 characters; the limit is 100') - fm = VALID_FRONTMATTER + "title: |\n Two\n lines\n" + fm = VALID_FRONTMATTER.replace("metadata:\n", "title: |\n Two\n lines\nmetadata:\n") self.assertOneError(tree(skill_md(fm)), '"title" must be a single line') @@ -288,7 +381,7 @@ def test_metadata_must_be_a_mapping(self): self.assertOneError(tree(skill_md(fm)), '"metadata" must be a mapping', line=4) def test_metadata_values_must_be_strings(self): - fm = frontmatter_with(version="1.0") + fm = frontmatter_with(version="1.5") error = self.assertOneError(tree(skill_md(fm)), '"metadata.version" must be text, not a number', line=6) self.assertIn("quotes", error.message) @@ -487,15 +580,17 @@ def test_a_key_with_no_value_says_so(self): error = self.assertOneError(tree(skill_md(frontmatter_with(author=""))), '"metadata.author" has no value') self.assertNotIn("remove the key", error.message) - def test_special_yaml_values_are_named(self): - fm = frontmatter_with().replace("metadata:\n", "license: =\nmetadata:\n") - self.assertOneError(tree(skill_md(fm)), '"license" must be text, not a special YAML value') + def test_unquoted_merge_and_value_indicators_are_rejected(self): + for value in ("=", "<<"): + with self.subTest(value=value): + fm = frontmatter_with().replace("metadata:\n", f"license: {value}\nmetadata:\n") + self.assertOneError(tree(skill_md(fm)), f"the unquoted value {value}, which YAML parsers", line=4) def test_values_other_parsers_read_as_numbers_must_be_quoted(self): for value in ("1e3", "-1.0e3", "+.5", "0o17", "+0o17", "1_0e3", "._5", "1_0.5e3"): with self.subTest(value=value): fm = frontmatter_with().replace("metadata:\n", f"title: {value}\nmetadata:\n") - self.assertOneError(tree(skill_md(fm)), "other YAML parsers read") + self.assertOneError(tree(skill_md(fm)), "don't all read as the same type") fm = frontmatter_with().replace("metadata:\n", 'title: "1e3"\nmetadata:\n') self.assertEqual(self.errors(tree(skill_md(fm))), []) diff --git a/tests/test_validate_skills.py b/tests/test_validate_skills.py index 1a21fddc2..6a6b4e6cb 100644 --- a/tests/test_validate_skills.py +++ b/tests/test_validate_skills.py @@ -367,34 +367,59 @@ def test_a_deprecated_skill_cannot_stay_recommended_by_a_connector(self): code, out = self.run_check("--base-ref", base) self.assertEqual(code, 0, out) - def test_a_connector_yaml_that_cannot_be_read_is_a_warning_not_a_crash(self): + def test_a_connector_yaml_that_breaks_the_yaml_rules_fails_and_an_odd_shape_warns(self): self.write("skills/alpha/SKILL.md", skill_md("alpha")) base = self.commit("base") cases = { - "skills: [alpha\n": "can't be read as YAML", - "skills: " + "[" * 2000 + "]" * 2000 + "\n": "can't be read as YAML", - "name: acme\nreleased: 2024-13-45\nskills: [alpha]\n": None, - "port: !!int abc\nskills: [alpha]\n": None, - "name: acme\nskills: alpha\n": "has a skills: value that isn't a list", - "- alpha\n": "isn't a YAML mapping", + "skills: [alpha\n": "FAIL connectors/acme/connector.yaml:2: connector.yaml is not valid YAML", + "skills: " + "[" * 2000 + "]" * 2000 + "\n": "FAIL connectors/acme/connector.yaml:1: connector.yaml is nested too deeply", + "name: acme\nreleased: 2024-13-45\nskills: [alpha]\n": "FAIL connectors/acme/connector.yaml:2: connector.yaml has the unquoted value 2024-13-45", + "port: !!int abc\nskills: [alpha]\n": "FAIL connectors/acme/connector.yaml:1: connector.yaml uses tags", + "name: acme\nskills: alpha\n": "WARN connectors/acme/connector.yaml: has a skills: value that isn't a list", + "- alpha\n": "WARN connectors/acme/connector.yaml: isn't a YAML mapping", } - for text, warning in cases.items(): + for text, expected in cases.items(): with self.subTest(text=text[:24]): self.write("connectors/acme/connector.yaml", text) self.commit("an odd connector") code, out = self.run_check("--base-ref", base) - self.assertEqual(code, 0, out) - if warning: - self.assertIn(f"WARN connectors/acme/connector.yaml: {warning}", out) - else: - self.assertNotIn("connectors/acme", out) + self.assertEqual(code, 1 if expected.startswith("FAIL") else 0, out) + self.assertIn(expected, out) + + def test_a_connector_yaml_over_32_kib_fails(self): + self.write("skills/alpha/SKILL.md", skill_md("alpha")) + text = "name: acme\n# " + "x" * 32_768 + "\n" + self.write("connectors/acme/connector.yaml", text) + self.commit("a large connector") + code, out = self.run_check() + self.assertEqual(code, 1, out) + self.assertIn(f"FAIL connectors/acme/connector.yaml: connector.yaml is {len(text):,} bytes; the limit is 32,768", out) + + def test_a_connector_yaml_that_is_not_utf8_fails(self): + self.write("skills/alpha/SKILL.md", skill_md("alpha")) + (self.repo / "connectors" / "acme").mkdir(parents=True) + (self.repo / "connectors" / "acme" / "connector.yaml").write_bytes(b"name: acme\ntitle: \xff\n") + self.commit("a connector that isn't UTF-8") + code, out = self.run_check() + self.assertEqual(code, 1, out) + self.assertIn("FAIL connectors/acme/connector.yaml:2: connector.yaml is not valid UTF-8", out) + + def test_a_releases_file_that_breaks_the_yaml_rules_fails(self): + self.write("skills/alpha/SKILL.md", skill_md("alpha")) + self.write("connectors/releases.yaml", "schema_version: 1\nconnectors: []\n") + self.commit("base") + code, out = self.run_check() + self.assertEqual(code, 0, out) + self.write("connectors/releases.yaml", "schema_version: 1\nconnectors: [acme,\n bronto]\n") + self.commit("a flow list over two lines") + code, out = self.run_check() + self.assertEqual(code, 1, out) + self.assertIn("FAIL connectors/releases.yaml:2: releases.yaml has a flow collection", out) def test_odd_connector_lists_still_find_the_skill(self): cases = { "name: acme\nskills: [alpha, null]\n": "has a skills: entry that isn't a skill name", - "defaults: &d\n skills: [alpha]\nname: acme\n<<: *d\n": None, - "skills: []\nname: acme\nskills: [alpha]\n": "has more than one skills: key", - "defaults: &d\n skills: []\nname: acme\nskills: [alpha]\n<<: *d\n": None, + "name: acme\nskills:\n - 'alpha'\n": None, } for text, warning in cases.items(): with self.subTest(text=text[:24]): @@ -411,16 +436,6 @@ def test_odd_connector_lists_still_find_the_skill(self): else: self.assertNotIn("WARN connectors/", out) - def test_values_pyyaml_cannot_build_still_find_the_skill(self): - self.write("skills/alpha/SKILL.md", skill_md("alpha")) - self.write("connectors/acme/connector.yaml", "name: acme\nreleased: 2024-13-45\nskills: [alpha]\n") - base = self.commit("base") - self.write("skills/alpha/SKILL.md", skill_md("alpha", version="1.0.1", extra=' deprecated: "true"\n')) - self.commit("deprecate alpha") - code, out = self.run_check("--base-ref", base) - self.assertEqual(code, 1, out) - self.assertIn("connectors/acme/connector.yaml still recommends it", out) - def test_a_skill_removed_through_a_merge_commit_is_retired(self): self.write("skills/alpha/SKILL.md", skill_md("alpha")) self.write("skills/old/SKILL.md", skill_md("old")) From 73165dd97329031e27991614283e4c78d7eb454e Mon Sep 17 00:00:00 2001 From: miscreantmoogly Date: Tue, 10 Nov 2026 23:57:20 -0700 Subject: [PATCH 2/6] feat(ci): check connector versions and breaking changes in PRs validate-skills didn't look at connector versions, so two connector changes passed every check and then stopped the catalog publisher, which refuses the whole run while any connector breaks its version rules: a logo change at an already-published version, and an auth header label renamed within the same major version. A replay of every merge on main with this check flags exactly those two. Add .github/scripts/connector_checks.py, which compares each connectors// folder with its merge-base copy the way the publisher compares a connector with its published version: - a connector folder is never removed (deprecate it instead), and its name and type never change - every file in the folder is published, so any change to one needs a higher version, and the version never goes down - within a major version, no mcp.auth option is removed or changed (any field, a header label included), no endpoint is removed or gets another URL, and no mcp.config field is added - connector.yaml has a MAJOR.MINOR.PATCH version, and a new connector not at 1.0.0 or a version that skips a step is a warning connector.yaml is compared as the publisher reads it: plain true and false are booleans, safe integers and shortest-form decimals are numbers, and the rest is text, so comments and quoting never count as a major change. skill_checks.plain_scalar_value holds that typing next to the YAML rules it depends on. The rules work on the same in-memory SkillTree as the skill rules and share their helpers: format_version and next_versions become public, the lenient PyYAML reader becomes compose_lenient, and run_cases holds the pass/fail matching for both case files. validate_skills.py reads the connector folders with one ls-tree reader, which its other three listings now use too, reports each finding under connectors//connector.yaml at the version line, and runs the new skill-rules/connector-conformance-cases.json before checking anything. Connector findings always show, since a warning can only come from a folder that differs from its merge-base copy. A connector.yaml that isn't a mapping is now an error, since it has no version to compare, and no longer also a warning from the skills: check. The connector fixtures in tests/test_validate_skills.py gain a version, and dropping a skill from a connector's list now bumps the connector, as the publisher requires. CONTRIBUTING.md, the connector review steering doc, the PR template's Connectors note, the workflow header and the project conventions describe the rules. --- .claude/CLAUDE.md | 5 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/scripts/connector_checks.py | 283 ++++++++++++++ .../connector-conformance-cases.json | 361 ++++++++++++++++++ .github/scripts/skill_checks.py | 74 +++- .github/scripts/validate_skills.py | 102 +++-- .github/workflows/validate-skills.yml | 10 +- .kiro/steering/connector-review.md | 7 +- .kiro/steering/project-conventions.md | 5 +- CONTRIBUTING.md | 25 +- tests/test_connector_checks.py | 284 ++++++++++++++ tests/test_validate_skills.py | 97 ++++- 12 files changed, 1178 insertions(+), 77 deletions(-) create mode 100644 .github/scripts/connector_checks.py create mode 100644 .github/scripts/skill-rules/connector-conformance-cases.json create mode 100644 tests/test_connector_checks.py diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 54d1f1ed6..c84f10d53 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -92,7 +92,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar `.github/workflows/validate-skills.yml` runs `.github/scripts/validate_skills.py` on every pull request, and `.github/workflows/validate-skills-main.yml` runs it again after each push to `main`. `.github/workflows/build-docs.yml` runs `mkdocs build --strict` on pull requests and on `main`. The rules and their reasons are in the "Skill Publishing Rules" section of [CONTRIBUTING.md](../CONTRIBUTING.md). -- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. +- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. - Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. @@ -101,7 +101,8 @@ The `metadata` block with `author` and `version` fields is required, and `summar - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. - `check_added_content` warns about internal host or tool names only on lines a PR adds, like `scan-aws-identifiers`, so names already on `main` don't warn on every PR. - With `--base-ref`, `check_history` compares each skill with its merge-base copy: no removed folders, no reuse of a path deleted in `main`'s first-parent history (`git log --first-parent --no-renames --diff-filter=D`, skipped with a warning on a shallow clone), a higher version whenever `content_hash` of the published files changes, one step at a time (next patch, minor or major), and `1.0.0` for a new skill. No folder is ever deleted, not even in an emergency: deprecate it instead. The hash function is pinned by the golden fixture under `content_hash` in `conformance-cases.json`, so any other implementation can prove it computes the same hash. -- With or without a base, a deprecated skill can't stay in any `connectors/*/connector.yaml` `skills:` list (`connector_skill_references`). Every `connectors/*/connector.yaml` and `connectors/releases.yaml` is read with `skill_checks.read_yaml`, the reader the frontmatter uses, so a file that isn't UTF-8 or breaks a YAML rule is an error; the cases are in `skill-rules/yaml-conformance-cases.json`, which `tests/test_skill_checks.py` runs. A `connector.yaml` that isn't a mapping or has a `skills:` value that isn't a list of skill names gets a warning; the names it does list are still checked. +- With or without a base, a deprecated skill can't stay in any `connectors/*/connector.yaml` `skills:` list (`connector_skill_references`). Every `connectors/*/connector.yaml` and `connectors/releases.yaml` is read with `skill_checks.read_yaml`, the reader the frontmatter uses, so a file that isn't UTF-8 or breaks a YAML rule is an error; the cases are in `skill-rules/yaml-conformance-cases.json`, which `tests/test_skill_checks.py` runs. A `connector.yaml` that has a `skills:` value that isn't a list of skill names gets a warning; the names it does list are still checked. One that isn't a mapping is an error from the connector version rules, since it has no version. +- `connector_checks.check_history` compares each `connectors//` folder (not `connectors/schema/`) with its merge-base copy, as the catalog publisher compares a connector with its published version, and reads `connector.yaml` as the publisher does (`skill_checks.plain_scalar_value`); without a base only `check_connector` runs. The rules are under "Connector versions" in CONTRIBUTING.md. - Secrets in test fixtures are built at runtime (`"AKIA" + ...`) or stored as base64 in the conformance cases, so no file in the repository contains a literal key or invisible character. Internal names in tests are made up (`corp.example`, `Rivendell`), never real hosts or tools. - `conformance-cases.json` holds language-neutral cases for these rules. Another tool that validates or publishes these skills should give the same `expect` result on the cases for the rules it implements; some rules here are stricter than any other tool's. The script runs every case before checking any skill and exits 2 on a mismatch. A rule change needs a matching case change in the same PR. - Every rule applies to every skill; nothing is exempted. A rule that existing skills don't meet yet starts as a warning and becomes an error after a cleanup PR brings every skill into line: a missing `metadata.summary`, and dimension values outside the vocabulary. A missing `metadata.aws-devops-agent-skills.agent-types` stays a warning. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 800839357..48d052b83 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -94,7 +94,7 @@ before this check existed" in CONTRIBUTING.md. ## Connectors — complete if this PR adds or changes a connector - + - [ ] **I accept the [AWS DevOps Agent Partner Connector Terms and Conditions](https://github.com/aws/tools-for-devops-agent/blob/main/connectors/TERMS.md), version 2026-11-03, on behalf of the company that owns this connector, and I have authority to do so.** diff --git a/.github/scripts/connector_checks.py b/.github/scripts/connector_checks.py new file mode 100644 index 000000000..0da992175 --- /dev/null +++ b/.github/scripts/connector_checks.py @@ -0,0 +1,283 @@ +"""The connector version rules: each connectors// folder compared with its merge-base copy. + +The rules and their reasons are under "Connector versions" in the "Skill Publishing +Rules" section of CONTRIBUTING.md. Like skill_checks.py, every function works on an +in-memory folder (sc.SkillTree), so the same rules run on Git objects in +validate_skills.py and on the cases in skill-rules/connector-conformance-cases.json. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field + +from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode + +import skill_checks as sc + +CONNECTORS_DIR = "connectors" +CONNECTOR_FILE = "connector.yaml" +# connectors/schema/ holds the JSON Schemas, not a connector. +SCHEMA_FOLDER = "schema" +CONFORMANCE_FILE = "connector-conformance-cases.json" + + +class _Absent: + """Stands for a key a document doesn't have, which equals only itself.""" + + def __repr__(self) -> str: + return "absent" + + +ABSENT = _Absent() + + +@dataclass(frozen=True) +class Document: + """connector.yaml as JSON, and the line of each top-level key.""" + + value: object + lines: dict[str, int] = field(default_factory=dict) + + def get(self, key: str) -> object: + return self.value.get(key, ABSENT) if isinstance(self.value, dict) else ABSENT + + +def read_document(data: bytes | None, *, strict: bool = True) -> Document | None: + """connector.yaml as the catalog publisher reads it, or None when it can't be read. + + A file with no YAML document, such as an empty one or one of only comments, + reads as null, which isn't a mapping. strict=False also reads a merge-base + copy that breaks a YAML rule added since it merged, as PyYAML alone reads it. + """ + if data is None: + return None + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + return None + root, problems = sc.read_yaml(text, CONNECTOR_FILE, first_line=1) + if problems and not strict and (lenient := sc.compose_lenient(text)) is not None: + root, problems = lenient, [] + if problems: + return None + if root is None: + return Document(None) + try: + value = to_json(root) + except RecursionError: + return None + lines = ( + {key.value: key.start_mark.line + 1 for key, _ in root.value if isinstance(key, ScalarNode)} + if isinstance(root, MappingNode) + else {} + ) + return Document(value, lines) + + +def to_json(node: Node) -> object: + """A YAML node as JSON. Only plain scalars are typed; a repeated key keeps its last value, as PyYAML does.""" + if isinstance(node, ScalarNode): + return sc.plain_scalar_value(node.value) if node.style is None else node.value + if isinstance(node, SequenceNode): + return [to_json(item) for item in node.value] + if isinstance(node, MappingNode): + return {str(key.value): to_json(value) for key, value in node.value} + raise TypeError(f"unexpected YAML node {type(node).__name__}") + + +def same(left: object, right: object) -> bool: + """Whether two to_json values are equal as JavaScript compares them deeply: true is not 1, and 1 is not "1". + + Comparing their JSON text gives that, because to_json never makes an integral + float: sc.plain_scalar_value reads 1.0 as text, so equal numbers have equal text. + """ + return left is right or ( + ABSENT not in (left, right) and json.dumps(left, sort_keys=True) == json.dumps(right, sort_keys=True) + ) + + +def _version(document: Document | None) -> tuple[int, int, int] | None: + version = document.get("version") if document is not None else ABSENT + return sc.parse_version(version) if isinstance(version, str) else None + + +def _show(value: object) -> str: + return "nothing" if value is ABSENT else json.dumps(value, ensure_ascii=False) + + +def _data(folder: sc.SkillTree | None, path: str) -> bytes | None: + entry = folder.entries.get(path) if folder is not None else None + return entry.data if entry is not None else None + + +def _hash(folder: sc.SkillTree) -> str: + """sc.content_hash of every file in the folder, since every one is published.""" + return sc.content_hash({path: entry.data for path, entry in folder.entries.items()}) + + +def check_connector(name: str, folder: sc.SkillTree) -> list[sc.Finding]: + """The rules that need only this copy: connector.yaml is there and has a version the other rules can compare. + + YAML problems in connector.yaml are left out: validate_skills.py reports + them for every file under connectors/. + """ + data = _data(folder, CONNECTOR_FILE) + if data is None: + message = f"{CONNECTORS_DIR}/{name}/ has no {CONNECTOR_FILE}; every connector folder needs one" + return [sc.Finding("error", message)] + document = read_document(data) + if document is None: + return [] + if not isinstance(document.value, dict): + message = f"{CONNECTOR_FILE} must be a mapping of keys to values, such as name: {name}" + return [sc.Finding("error", message, CONNECTOR_FILE, 1)] + if _version(document) is not None: + return [] + version = document.get("version") + if version is ABSENT: + return [sc.Finding("error", '"version" is required, such as version: 1.0.0', CONNECTOR_FILE)] + shown = _show(version[:40]) if isinstance(version, str) else _show(version)[:40] + message = ( + f'"version" is {shown}; it must be MAJOR.MINOR.PATCH, such as version: 1.0.0, with at most ' + f"{sc.MAX_VERSION_PART_DIGITS} digits in each part" + ) + return [sc.Finding("error", message, CONNECTOR_FILE, document.lines.get("version"))] + + +def check_history(name: str, base: sc.SkillTree | None, head: sc.SkillTree | None) -> list[sc.Finding]: + """The rules that compare a connector folder with its merge-base copy. None means the folder isn't there.""" + if base is not None and head is None: + return [ + sc.Finding( + "error", + f"the connector folder {CONNECTORS_DIR}/{name} was removed. A published connector stays in the " + f"catalog, so its folder must stay too: set deprecated: true in its {CONNECTOR_FILE} instead", + ) + ] + if head is None: + return [] + document = read_document(_data(head, CONNECTOR_FILE)) + version = _version(document) + if version is None: + return [] # check_connector reports why there is no version to compare. + version_line = document.lines.get("version") + if base is None: + if version != sc.FIRST_VERSION: + message = f"a new connector starts at version 1.0.0, not {sc.format_version(version)}" + return [sc.Finding("warning", message, CONNECTOR_FILE, version_line)] + return [] + old_document = read_document(_data(base, CONNECTOR_FILE), strict=False) + old_version = _version(old_document) + if old_version is None: + message = ( + f"the version rules were not checked, because {CONNECTOR_FILE} on the base branch has no version that " + "can be read" + ) + return [sc.Finding("warning", message, CONNECTOR_FILE, version_line)] + + findings = [] + for key in ("name", "type"): + old, new = old_document.get(key), document.get(key) + if not same(old, new): + message = ( + f'"{key}" changed from {_show(old)} to {_show(new)}; a connector\'s {key} never changes, so set it back' + ) + findings.append(sc.Finding("error", message, CONNECTOR_FILE, document.lines.get(key, version_line))) + + def error(message: str) -> None: + findings.append(sc.Finding("error", message, CONNECTOR_FILE, version_line)) + + old_text, new_text = sc.format_version(old_version), sc.format_version(version) + if version < old_version: + error(f"version went down from {old_text} to {new_text}; it must never go down") + elif version == old_version: + if _hash(base) != _hash(head): + error( + f"files in {CONNECTORS_DIR}/{name}/ changed, so version must go up from {old_text}. Every file in a " + "connector folder is published, and a published version never changes" + ) + else: + if version[0] == old_version[0]: + for reason in major_change_reasons(old_document.value, document.value): + error( + f"{reason}, which needs a new major version: {version[0] + 1}.0.0, not {new_text}. Within a " + "major version, auth options and endpoint URLs stay as they are and no config field is added, " + "so that connections customers already made keep working" + ) + if version not in sc.next_versions(old_version): + steps = [sc.format_version(step) for step in sc.next_versions(old_version)] + message = ( + f"version should go up only one step from {old_text}: to {steps[0]}, {steps[1]} or {steps[2]}, " + f"not {new_text}" + ) + findings.append(sc.Finding("warning", message, CONNECTOR_FILE, version_line)) + return findings + + +def _objects(value: object) -> list[dict]: + """The items of a JSON list that are objects. Anything else is a schema problem, not a version one.""" + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + +def _mcp(document: object) -> dict: + mcp = document.get("mcp") if isinstance(document, dict) else None + return mcp if isinstance(mcp, dict) else {} + + +def _auth_options(document: object) -> dict[str, dict]: + """Each mcp.auth option by its type. The last one wins if a type repeats, which the schema forbids anyway.""" + options = _objects(_mcp(document).get("auth")) + return {option["type"]: option for option in options if isinstance(option.get("type"), str)} + + +def _endpoint_urls(document: object) -> dict[str, object]: + """Each endpoint's URL, named for the message: mcp.endpoint, or an mcp.endpoints entry by its label.""" + mcp = _mcp(document) + urls: dict[str, object] = {} + if "endpoint" in mcp: + urls["mcp.endpoint"] = mcp["endpoint"] + for entry in _objects(mcp.get("endpoints")): + if isinstance(entry.get("label"), str): + urls[f'the mcp.endpoints entry "{entry["label"]}"'] = entry.get("endpoint", ABSENT) + return urls + + +def _config_field_names(document: object) -> list[str]: + return [item["name"] for item in _objects(_mcp(document).get("config")) if isinstance(item.get("name"), str)] + + +def major_change_reasons(old: object, new: object) -> list[str]: + """What changed from `old` to `new` that only a major version may change, as the catalog publisher counts it.""" + reasons = [] + new_options = _auth_options(new) + for kind, option in _auth_options(old).items(): + if not same(option, new_options.get(kind, ABSENT)): + reasons.append(f'the "{kind}" option in mcp.auth was removed or changed') + new_urls = _endpoint_urls(new) + for endpoint, url in _endpoint_urls(old).items(): + if not same(url, new_urls.get(endpoint, ABSENT)): + reasons.append(f"{endpoint} was removed or its URL changed") + old_fields = set(_config_field_names(old)) + for name in dict.fromkeys(_config_field_names(new)): + if name not in old_fields: + reasons.append(f'the mcp.config field "{name}" was added') + return reasons + + +# --- Conformance cases ------------------------------------------------------------ + + +def run_conformance(cases: list[dict]) -> list[str]: + """Problems with the rules' results on the cases from sc.load_cases; empty when every case gives its result.""" + return sc.run_cases(cases, _case_findings, label="connector case") + + +def _case_findings(case: dict) -> list[sc.Finding]: + name = case["name"] + head = sc.tree_from_files(name, case["files"]) if case.get("files") is not None else None + findings = check_connector(name, head) if head is not None else [] + if "base" in case: + base = sc.tree_from_files(name, case["base"]) if case.get("base") is not None else None + findings += check_history(name, base, head) + return findings diff --git a/.github/scripts/skill-rules/connector-conformance-cases.json b/.github/scripts/skill-rules/connector-conformance-cases.json new file mode 100644 index 000000000..9c827a584 --- /dev/null +++ b/.github/scripts/skill-rules/connector-conformance-cases.json @@ -0,0 +1,361 @@ +{ + "description": "Conformance cases for the connector version rules in connector_checks.py. Each case is one connector folder, connectors//: base is its copy on the base branch, which stands for the published version, and files is the pull request's copy. base: null means the pull request adds the connector, and files: null means it removes the folder. expect is this check's result: pass means no errors (warnings are allowed), fail means at least one error, and error is a fragment of this repository's own message. A file is given as UTF-8 text or as base64 bytes. The catalog publisher applies the same rules when it publishes a connector, so another tool that checks these folders should give the same result on the cases for the rules it implements. The fixtures are not complete connectors: their connector.yaml lacks fields the schema requires, and a case holds no other file unless its rule is about one. validate_skills.py runs every case before it checks anything, and tests/test_connector_checks.py runs them too.", + "cases": [ + { + "id": "auth-label-changed-in-a-major", + "name": "acme", + "expect": "pass", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.1\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: Personal API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 2.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: Personal or team API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "auth-label-changed-in-a-patch", + "name": "acme", + "expect": "fail", + "error": "needs a new major version", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.1\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: Personal API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.2\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: Personal or team API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "auth-option-added-in-a-minor", + "name": "acme", + "expect": "pass", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.1.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: oauth_client_credentials\n exchange_url: \"https://login.acme.example/token\"\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "auth-value-type-changed-in-a-minor", + "name": "acme", + "expect": "fail", + "error": "needs a new major version", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoints:\n - { label: EU, endpoint: \"https://eu.acme.example/mcp\" }\n - { label: US, endpoint: \"https://us.acme.example/mcp\" }\n auth:\n - type: oauth_3lo\n authorization_url: \"https://login.acme.example/authorize\"\n exchange_url: \"https://login.acme.example/token\"\n pkce: true\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.1.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoints:\n - { label: EU, endpoint: \"https://eu.acme.example/mcp\" }\n - { label: US, endpoint: \"https://us.acme.example/mcp\" }\n auth:\n - type: oauth_3lo\n authorization_url: \"https://login.acme.example/authorize\"\n exchange_url: \"https://login.acme.example/token\"\n pkce: \"true\"\n" + } + } + }, + { + "id": "comment-changed-at-the-same-version", + "name": "acme", + "expect": "fail", + "error": "version must go up from", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "# Acme's hosted MCP server.\nschema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "comment-changed-with-a-patch", + "name": "acme", + "expect": "pass", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "# Acme's hosted MCP server.\nschema_version: 1\nname: acme\nversion: 1.0.1\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "config-field-added-in-a-minor", + "name": "acme", + "expect": "fail", + "error": "the mcp.config field \"team\" was added", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoints:\n - { label: EU, endpoint: \"https://eu.acme.example/mcp\" }\n - { label: US, endpoint: \"https://us.acme.example/mcp\" }\n auth:\n - type: oauth_3lo\n authorization_url: \"https://login.acme.example/authorize\"\n exchange_url: \"https://login.acme.example/token\"\n pkce: true\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.1.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoints:\n - { label: EU, endpoint: \"https://eu.acme.example/mcp\" }\n - { label: US, endpoint: \"https://us.acme.example/mcp\" }\n auth:\n - type: oauth_3lo\n authorization_url: \"https://login.acme.example/authorize\"\n exchange_url: \"https://login.acme.example/token\"\n pkce: true\n config:\n - { name: team, label: Team, pattern: \"^[a-z0-9-]{1,32}$\" }\n" + } + } + }, + { + "id": "connector-yaml-empty", + "name": "acme", + "expect": "fail", + "error": "must be a mapping of keys to values", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "" + } + } + }, + { + "id": "connector-yaml-missing", + "name": "acme", + "expect": "fail", + "error": "has no connector.yaml", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + }, + "README.md": { + "text": "# Acme\n\nConnect Acme to AWS DevOps Agent.\n" + } + }, + "files": { + "README.md": { + "text": "# Acme\n\nConnect Acme to AWS DevOps Agent.\n" + } + } + }, + { + "id": "endpoint-url-changed-in-a-minor", + "name": "acme", + "expect": "fail", + "error": "needs a new major version", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.1.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/v2/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "endpoints-entry-url-changed-in-a-minor", + "name": "acme", + "expect": "fail", + "error": "the mcp.endpoints entry \"US\" was removed or its URL changed", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoints:\n - { label: EU, endpoint: \"https://eu.acme.example/mcp\" }\n - { label: US, endpoint: \"https://us.acme.example/mcp\" }\n auth:\n - type: oauth_3lo\n authorization_url: \"https://login.acme.example/authorize\"\n exchange_url: \"https://login.acme.example/token\"\n pkce: true\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.1.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoints:\n - { label: EU, endpoint: \"https://eu.acme.example/mcp\" }\n - { label: US, endpoint: \"https://us.acme.example/v2/mcp\" }\n auth:\n - type: oauth_3lo\n authorization_url: \"https://login.acme.example/authorize\"\n exchange_url: \"https://login.acme.example/token\"\n pkce: true\n" + } + } + }, + { + "id": "endpoints-reordered-in-a-patch", + "name": "acme", + "expect": "pass", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoints:\n - { label: EU, endpoint: \"https://eu.acme.example/mcp\" }\n - { label: US, endpoint: \"https://us.acme.example/mcp\" }\n auth:\n - type: oauth_3lo\n authorization_url: \"https://login.acme.example/authorize\"\n exchange_url: \"https://login.acme.example/token\"\n pkce: true\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.1\ntitle: Acme\ntype: mcp\nmcp:\n endpoints:\n - { label: US, endpoint: \"https://us.acme.example/mcp\" }\n - { label: EU, endpoint: \"https://eu.acme.example/mcp\" }\n auth:\n - type: oauth_3lo\n authorization_url: \"https://login.acme.example/authorize\"\n exchange_url: \"https://login.acme.example/token\"\n pkce: true\n" + } + } + }, + { + "id": "folder-removed", + "name": "acme", + "expect": "fail", + "error": "set deprecated: true", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": null + }, + { + "id": "logo-changed-at-the-same-version", + "name": "acme", + "expect": "fail", + "error": "version must go up from", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + }, + "logo.svg": { + "text": "\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + }, + "logo.svg": { + "text": "\n" + } + } + }, + { + "id": "name-changed", + "name": "acme", + "expect": "fail", + "error": "\"name\" changed from \"acme\" to \"acme-cloud\"", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme-cloud\nversion: 1.0.1\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "new-connector", + "name": "acme", + "expect": "pass", + "base": null, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "new-connector-not-at-1.0.0-warns-only", + "name": "acme", + "expect": "pass", + "base": null, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.2.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "quoting-changed-with-a-patch", + "name": "acme", + "expect": "pass", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.1\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: https://mcp.acme.example/mcp\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "type-changed", + "name": "acme", + "expect": "fail", + "error": "\"type\" changed from \"mcp\" to \"rest\"", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 2.0.0\ntitle: Acme\ntype: rest\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "unchanged", + "name": "acme", + "expect": "pass", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "version-jumps-far-warns-only", + "name": "acme", + "expect": "pass", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.3.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "version-not-major-minor-patch", + "name": "acme", + "expect": "fail", + "error": "must be MAJOR.MINOR.PATCH", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.0.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: \"1.1\"\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + }, + { + "id": "version-went-down", + "name": "acme", + "expect": "fail", + "error": "went down from 1.2.0 to 1.1.0", + "base": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.2.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + }, + "files": { + "connector.yaml": { + "text": "schema_version: 1\nname: acme\nversion: 1.1.0\ntitle: Acme\ntype: mcp\nmcp:\n endpoint: \"https://mcp.acme.example/mcp\"\n auth:\n - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: \"Bearer \" }\n" + } + } + } + ] +} diff --git a/.github/scripts/skill_checks.py b/.github/scripts/skill_checks.py index 50943bf35..d61765007 100644 --- a/.github/scripts/skill_checks.py +++ b/.github/scripts/skill_checks.py @@ -22,6 +22,7 @@ import zipfile from dataclasses import dataclass, field from pathlib import Path +from typing import Callable import yaml from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode @@ -736,6 +737,23 @@ def _plain_scalar_problem(text: str, is_in_flow_collection: bool, *, is_key: boo ) +def plain_scalar_value(text: str) -> bool | int | float | str | None: + """The JSON value of a plain (unquoted) scalar that read_yaml accepts, as every parser types it. + + true and false are booleans; null, ~ and an empty value are null; an + integer in JavaScript's safe range, or a decimal written the shortest way, + is a number; anything else is text. read_yaml refuses every other plain + value that a parser reads as something other than text. + """ + if text in ("true", "false"): + return text == "true" + if text in ("null", "~", ""): + return None + if _is_typed_alike_by_every_parser(text): + return int(text) if CANONICAL_INTEGER_PATTERN.fullmatch(text) else float(text) + return text + + def _is_read_as_non_text_by_some_parser(text: str) -> bool: return any(pattern.fullmatch(text) for pattern in NON_TEXT_PLAIN_SCALAR_PATTERNS) @@ -855,10 +873,7 @@ def _parse_frontmatter(frontmatter: str, report: _Report, *, strict: bool = True that may predate the stricter rules. """ if not strict: - try: - return yaml.compose(frontmatter, Loader=yaml.SafeLoader) - except (yaml.YAMLError, RecursionError): - return None + return compose_lenient(frontmatter) root, problems = read_yaml(frontmatter, "the frontmatter", FRONTMATTER_FIRST_LINE, FRONTMATTER_FORBIDDEN_TOKENS) for line, message in problems: report.error(message, line=line) @@ -878,6 +893,14 @@ def _parse_frontmatter(frontmatter: str, report: _Report, *, strict: bool = True return root +def compose_lenient(text: str) -> Node | None: + """The node tree as PyYAML alone reads it, or None, for a merge-base copy that may predate the stricter rules.""" + try: + return yaml.compose(text, Loader=yaml.SafeLoader) + except (yaml.YAMLError, RecursionError): + return None + + def _mapping_items(node: MappingNode) -> dict[str, tuple[ScalarNode, Node]]: """The key and value nodes by key. read_yaml has already refused keys that aren't text or that repeat.""" return {key_node.value: (key_node, value_node) for key_node, value_node in node.value} @@ -1549,11 +1572,11 @@ def published_hash(tree: SkillTree, rules: Rules) -> str: return content_hash({p: e.data for p, e in published_files(tree, rules).items()}) -def _format_version(version: tuple[int, int, int]) -> str: +def format_version(version: tuple[int, int, int]) -> str: return ".".join(str(part) for part in version) -def _next_versions(version: tuple[int, int, int]) -> tuple[tuple[int, int, int], ...]: +def next_versions(version: tuple[int, int, int]) -> tuple[tuple[int, int, int], ...]: """The only versions allowed after `version`: the next patch, minor or major release.""" major, minor, patch = version return ((major, minor, patch + 1), (major, minor + 1, 0), (major + 1, 0, 0)) @@ -1617,7 +1640,7 @@ def check_history( ) ] if info is not None and info.version is not None and info.version != FIRST_VERSION: - message = f"a new skill starts at version 1.0.0, not {_format_version(info.version)}" + message = f"a new skill starts at version 1.0.0, not {format_version(info.version)}" return [Finding("error", message, "SKILL.md", info.version_line)] return [] @@ -1637,26 +1660,26 @@ def check_history( new = head_info.version changed = published_hash(base, rules) != published_hash(head, rules) if new < old: - message = f"metadata.version went down from {_format_version(old)} to {_format_version(new)}; it must never go down" + message = f"metadata.version went down from {format_version(old)} to {format_version(new)}; it must never go down" return [Finding("error", message, "SKILL.md", head_info.version_line)] if new == old and changed: message = ( - f"published files changed, so metadata.version must go up from {_format_version(old)}. " + f"published files changed, so metadata.version must go up from {format_version(old)}. " f"A published version is immutable. Changes only to {', '.join(rules.published_excludes)} need no bump" ) return [Finding("error", message, "SKILL.md", head_info.version_line)] - if new > old and new not in _next_versions(old): - steps = [_format_version(v) for v in _next_versions(old)] + if new > old and new not in next_versions(old): + steps = [format_version(v) for v in next_versions(old)] message = ( - f"metadata.version can only go up one step from {_format_version(old)}: to {steps[0]}, {steps[1]} " - f"or {steps[2]}, not {_format_version(new)}" + f"metadata.version can only go up one step from {format_version(old)}: to {steps[0]}, {steps[1]} " + f"or {steps[2]}, not {format_version(new)}" ) return [Finding("error", message, "SKILL.md", head_info.version_line)] if new > old: before = base.entries.get("CHANGELOG.md") after = head.entries.get("CHANGELOG.md") if after is None or (before is not None and before.data == after.data): - message = f"metadata.version went up to {_format_version(new)}, but CHANGELOG.md has no new entry for it" + message = f"metadata.version went up to {format_version(new)}, but CHANGELOG.md has no new entry for it" return [Finding("warning", message, "CHANGELOG.md")] return [] @@ -1665,7 +1688,7 @@ def check_history( def load_cases(path: Path) -> list[dict]: - """The cases in conformance-cases.json. Raises ValueError when the file is malformed.""" + """The cases in conformance-cases.json or connector-conformance-cases.json. Raises ValueError when malformed.""" document = _read_json(path) cases = document.get("cases") if isinstance(document, dict) else None if not isinstance(cases, list) or not cases: @@ -1692,7 +1715,7 @@ def _case_problem(case) -> str | None: if case["expect"] == "fail" and not isinstance(case.get("error"), str): return 'a "fail" case needs an "error" string' if "base" in case and case["base"] is not None and not isinstance(case["base"], dict): - return '"base" must be an object of files, or null for a skill the pull request adds' + return '"base" must be an object of files, or null for a folder the pull request adds' if "retired" in case and not isinstance(case["retired"], bool): return '"retired" must be true or false' files = case.get("files") @@ -1738,8 +1761,8 @@ def tree_from_files(name: str, files: dict) -> SkillTree: def run_conformance(cases: list[dict], rules: Rules) -> list[str]: """Problems with the rules' results on the cases; an empty list means every case gives its expected result.""" - problems = [] - for case in cases: + + def findings_for(case: dict) -> list[Finding]: name = case["name"] head = tree_from_files(name, case["files"]) if case.get("files") is not None else None findings = check_skill(head, rules)[0] if head is not None else [] @@ -1747,10 +1770,19 @@ def run_conformance(cases: list[dict], rules: Rules) -> list[str]: base = tree_from_files(name, case["base"]) if case.get("base") is not None else None retired = frozenset({name}) if case.get("retired") else frozenset() findings += check_history(name, base, head, rules, retired) - errors = [f.message for f in findings if f.severity == "error"] + return findings + + return run_cases(cases, findings_for) + + +def run_cases(cases: list[dict], findings_for: Callable[[dict], list[Finding]], label: str = "case") -> list[str]: + """Problems with `findings_for(case)` on each case: pass means no errors, fail an error holding the case's text.""" + problems = [] + for case in cases: + errors = [f.message for f in findings_for(case) if f.severity == "error"] if case["expect"] == "pass" and errors: - problems.append(f"case {case['id']}: expected pass, got: {'; '.join(errors)}") + problems.append(f"{label} {case['id']}: expected pass, got: {'; '.join(errors)}") elif case["expect"] == "fail" and not any(case["error"] in message for message in errors): got = "; ".join(errors) if errors else "no errors" - problems.append(f"case {case['id']}: expected an error containing {case['error']!r}, got: {got}") + problems.append(f"{label} {case['id']}: expected an error containing {case['error']!r}, got: {got}") return problems diff --git a/.github/scripts/validate_skills.py b/.github/scripts/validate_skills.py index b5c44ad65..8fbaae293 100644 --- a/.github/scripts/validate_skills.py +++ b/.github/scripts/validate_skills.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 -"""Check that every skill in skills/ can be published. +"""Check that every skill in skills/, and every connector in connectors/, can be published. Skills on `main` are published automatically, so this check applies the rules in skill_checks.py to every skill folder on every pull request, not only to the skills the pull request changes. See the "Skill Publishing Rules" section of CONTRIBUTING.md. +Connectors on `main` are published the same way, so every connectors// +folder is checked by the rules in connector_checks.py too. + Skills are read from Git objects, never from the working tree, so a run sees exactly what a commit contains (file modes included) and nothing a checkout or an editor added. Commit a change before checking it locally. @@ -34,16 +37,17 @@ import os import subprocess import sys +from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path import yaml +import connector_checks as cc import skill_checks as sc SKILLS_DIR = "skills" -CONNECTORS_DIR = "connectors" -CONNECTOR_FILE = "connector.yaml" +CONNECTORS_DIR, CONNECTOR_FILE = cc.CONNECTORS_DIR, cc.CONNECTOR_FILE MAX_CONNECTOR_FILE_BYTES = 32_768 RELEASES_PATH = f"{CONNECTORS_DIR}/releases.yaml" RULES_DIR = Path(__file__).resolve().parent / "skill-rules" @@ -87,18 +91,21 @@ def _decode_path(raw: bytes) -> str: return raw.decode("utf-8", "surrogateescape") +def _ls_tree(repo_root: Path, rev: str, *paths: str) -> Iterator[tuple[str, str, str, str]]: + """(mode, kind, oid, path) for each entry of `git ls-tree -r` at `rev`, under `paths` when there are any.""" + for record in _git(repo_root, "ls-tree", "-r", "-z", "--full-tree", rev, "--", *paths).split(b"\0"): + if record: + meta, _, raw_path = record.partition(b"\t") + mode, kind, oid = meta.decode("ascii").split(" ") + yield mode, kind, oid, _decode_path(raw_path) + + def read_skills(repo_root: Path, rev: str) -> tuple[dict[str, sc.SkillTree], list[SkillResult]]: """Every skill folder at `rev`, plus results for entries in skills/ that aren't skill folders.""" - listing = _git(repo_root, "ls-tree", "-r", "-z", "--full-tree", rev, "--", f"{SKILLS_DIR}/") trees: dict[str, sc.SkillTree] = {} loose: list[SkillResult] = [] blobs: list[tuple[str, str, str, str]] = [] # (skill, relative path, mode, oid) - for record in listing.split(b"\0"): - if not record: - continue - meta, _, raw_path = record.partition(b"\t") - mode, kind, oid = meta.decode("ascii").split(" ") - path = _decode_path(raw_path) + for mode, kind, oid, path in _ls_tree(repo_root, rev, f"{SKILLS_DIR}/"): _, _, rest = path.partition("/") skill, _, inner = rest.partition("/") if not inner: @@ -176,14 +183,8 @@ def repository_results(repo_root: Path, rev: str) -> list[SkillResult]: must contain no symbolic link, and a root .gitattributes setting can change the bytes that archive holds for a skill's files. """ - listing = _git(repo_root, "ls-tree", "-r", "-z", "--full-tree", rev) results: list[SkillResult] = [] - for record in listing.split(b"\0"): - if not record: - continue - meta, _, raw_path = record.partition(b"\t") - mode, kind, oid = meta.decode("ascii").split(" ") - path = _decode_path(raw_path) + for mode, kind, oid, path in _ls_tree(repo_root, rev): if path.startswith(f"{SKILLS_DIR}/"): continue # Checked per skill. if mode == "120000": @@ -246,23 +247,54 @@ def retired_skills(repo_root: Path, base: str, present: set[str]) -> frozenset[s return frozenset(names - present) +def read_connectors(repo_root: Path, rev: str) -> dict[str, sc.SkillTree]: + """Every connectors// folder at `rev`. + + Files directly in connectors/, such as releases.yaml, belong to no folder, + and connectors/schema/ holds the schemas, not a connector. + """ + blobs: list[tuple[str, str, str, str]] = [] # (connector, path in its folder, mode, oid) + for mode, kind, oid, path in _ls_tree(repo_root, rev, f"{CONNECTORS_DIR}/"): + parts = path.split("/", 2) + if kind == "blob" and len(parts) == 3 and parts[1] != cc.SCHEMA_FOLDER: + blobs.append((parts[1], parts[2], mode, oid)) + folders: dict[str, sc.SkillTree] = {} + for (name, inner, mode, _), data in zip(blobs, _read_blobs(repo_root, [blob[3] for blob in blobs])): + folders.setdefault(name, sc.SkillTree(name)).entries[inner] = sc.Entry(inner, mode, data) + return folders + + +def connector_results(repo_root: Path, base: str | None, head_ref: str) -> list[SkillResult]: + """The connector_checks.py rules for every connector folder, compared with the merge base when there is one. + + Every finding shows, as for the connector YAML: a warning can only come from + a folder that differs from its merge-base copy. + """ + head_folders = read_connectors(repo_root, head_ref) + base_folders = read_connectors(repo_root, base) if base is not None else {} + results: list[SkillResult] = [] + for name in sorted(set(head_folders) | set(base_folders)): + head = head_folders.get(name) + findings = cc.check_connector(name, head) if head is not None else [] + if base is not None: + findings += cc.check_history(name, base_folders.get(name), head) + if findings: + results.append(SkillResult(f"{CONNECTORS_DIR}/{name}", findings, is_skill=False, always_visible=True)) + return results + + def connector_skill_references(repo_root: Path, rev: str) -> tuple[dict[str, list[str]], list[SkillResult]]: """Skill name -> the connectors whose connector.yaml lists it under skills: (recommended skills). Also returns an error for each connector.yaml, and for connectors/releases.yaml, that isn't UTF-8 or that breaks the YAML rules of sc.read_yaml, since the - connectors are published from the same files. A connector.yaml that isn't a - mapping or has a skills: value that isn't a list of skill names gets a - warning, so a broken file doesn't silently skip the check. + connectors are published from the same files. A connector.yaml whose skills: + value isn't a list of skill names gets a warning, so a broken file doesn't + silently skip the check; the connector version rules report one that isn't a + mapping. """ - listing = _git(repo_root, "ls-tree", "-r", "-z", "--full-tree", rev, "--", f"{CONNECTORS_DIR}/") files: list[tuple[str, str]] = [] - for record in listing.split(b"\0"): - if not record: - continue - meta, _, raw_path = record.partition(b"\t") - _, kind, oid = meta.decode("ascii").split(" ") - path = _decode_path(raw_path) + for _, kind, oid, path in _ls_tree(repo_root, rev, f"{CONNECTORS_DIR}/"): parts = path.split("/") is_connector_file = len(parts) == 3 and parts[2] == CONNECTOR_FILE if kind == "blob" and (is_connector_file or path == RELEASES_PATH): @@ -297,9 +329,13 @@ def _read_connector_yaml(path: str, data: bytes) -> tuple[yaml.Node | None, list def _recommended_skills(root: yaml.Node | None) -> tuple[str | None, list[str]]: - """The skill names under a connector.yaml's skills: key, and what kept any of it from being checked.""" + """The skill names under a connector.yaml's skills: key, and what kept any of it from being checked. + + A root that isn't a mapping has no skills: key, and the connector version + rules already report it as an error. + """ if not isinstance(root, yaml.MappingNode): - return "isn't a YAML mapping, so its skills: list wasn't checked", [] + return None, [] skills_node = next((value for key, value in root.value if key.value == "skills"), None) if skills_node is None: return None, [] @@ -340,8 +376,9 @@ def check_repository( history[name] = sc.check_history(name, base_trees.get(name), head_trees.get(name), rules, retired) for name in sorted(set(base_trees) - set(head_trees)): results.append(SkillResult(f"{SKILLS_DIR}/{name}", history[name], is_skill=False)) - recommended, connector_results = connector_skill_references(repo_root, head_ref) - results += connector_results + recommended, connector_yaml_results = connector_skill_references(repo_root, head_ref) + results += connector_yaml_results + results += connector_results(repo_root, base, head_ref) for name in sorted(head_trees): findings, info = sc.check_skill(head_trees[name], rules) if base is not None: @@ -463,9 +500,10 @@ def self_check(rules_dir: Path, rules: sc.Rules) -> list[str]: try: cases = sc.load_cases(rules_dir / CONFORMANCE_FILE) golden_files, golden_hash = sc.load_golden_hash(rules_dir / CONFORMANCE_FILE) + connector_cases = sc.load_cases(rules_dir / cc.CONFORMANCE_FILE) except ValueError as exc: return [str(exc)] - problems = sc.run_conformance(cases, rules) + problems = sc.run_conformance(cases, rules) + cc.run_conformance(connector_cases) actual = sc.content_hash(golden_files) if actual != golden_hash: problems.append(f"the content hash of the golden fixture is {actual}, not {golden_hash}") diff --git a/.github/workflows/validate-skills.yml b/.github/workflows/validate-skills.yml index 6491b1eff..a467bbd73 100644 --- a/.github/workflows/validate-skills.yml +++ b/.github/workflows/validate-skills.yml @@ -1,9 +1,11 @@ name: Validate skills -# Gates pull requests on every skill in skills/ meeting the publishing rules in -# .github/scripts/skill_checks.py (see "Skill Publishing Rules" in -# CONTRIBUTING.md). Skills on `main` are published automatically, so the check -# applies the rules to every skill, not only the ones the pull request changes. +# Gates pull requests on every skill in skills/ and every connector in +# connectors/ meeting the publishing rules in .github/scripts/skill_checks.py and +# .github/scripts/connector_checks.py (see "Skill Publishing Rules" in +# CONTRIBUTING.md). Skills and connectors on `main` are published automatically, +# so the check applies the rules to all of them, not only the ones the pull +# request changes. # # It runs the unit tests for the check first, so a pull request that changes # the check is tested by the version it brings. diff --git a/.kiro/steering/connector-review.md b/.kiro/steering/connector-review.md index fe0bd7bab..835a1c53f 100644 --- a/.kiro/steering/connector-review.md +++ b/.kiro/steering/connector-review.md @@ -8,7 +8,7 @@ description: Review pull requests that add or change a partner connector under c A connector is `connectors//`: a `connector.yaml` describing a partner-hosted MCP server, a `logo.svg` (optional `logo-dark.svg`), and a `README.md` setup guide, plus the skills listed under `skills:`. It holds no server code. Several connector PRs are usually open at once and reviewed by different maintainers, so review every one the same way and report in the same shape. -Repository facts to re-check if anything looks different: `/connectors/` is owned by the admin team in `.github/CODEOWNERS`; `connectors/README.md` says submissions are not open yet; the PR checklist check deliberately skips `connectors/` but still holds any `skills/` in the PR to the skill checklist; there is no repository validator for connector files, so step 3 does that work. +Repository facts to re-check if anything looks different: `/connectors/` is owned by the admin team in `.github/CODEOWNERS`; `connectors/README.md` says submissions are not open yet; the PR checklist check holds a connector change to the Connectors section (the partner terms box and `contact_email`) and any `skills/` in the PR to the skill checklist; `validate_skills` checks each connector's YAML and version rules (3c and 3d) but not its schema or the published-file allowlist, which step 3a covers. Partners build to the Connector Guide for Partners until it moves into `CONTRIBUTING.md`. Once `CONTRIBUTING.md` has a connector guide, it takes precedence over this file wherever the two disagree. @@ -249,8 +249,9 @@ Record the command, exit status, test count, and warning count for each. The uni **3d. Versions.** - A new connector or skill normally starts at `1.0.0`. -- For an existing connector, any change to its files needs a higher `version`. -- For an existing skill, any change to published content needs a higher `metadata.version`. `validate_skills` checks skills, not connectors. +- For an existing connector, any change to its files needs a higher `version`. Changing or removing an auth option (a header label included), removing an endpoint or changing its URL, or adding an `mcp.config` field needs a new major version. +- For an existing skill, any change to published content needs a higher `metadata.version`. +- `validate_skills` in 3c checks both against the merge base, so treat any version error it reports as a blocker. It compares with `main`, not with the catalog: while publishing is stalled and `main` holds a version the catalog hasn't published, also compare the pull request with the version the catalog serves. **3e. Find the write-capable skills.** Use both searches; the second catches skills that describe writes in prose without naming a tool. diff --git a/.kiro/steering/project-conventions.md b/.kiro/steering/project-conventions.md index 078cb069d..931499bc9 100644 --- a/.kiro/steering/project-conventions.md +++ b/.kiro/steering/project-conventions.md @@ -92,7 +92,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar `.github/workflows/validate-skills.yml` runs `.github/scripts/validate_skills.py` on every pull request, and `.github/workflows/validate-skills-main.yml` runs it again after each push to `main`. `.github/workflows/build-docs.yml` runs `mkdocs build --strict` on pull requests and on `main`. The rules and their reasons are in the "Skill Publishing Rules" section of [CONTRIBUTING.md](../../CONTRIBUTING.md). -- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. +- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. - Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. @@ -101,7 +101,8 @@ The `metadata` block with `author` and `version` fields is required, and `summar - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. - `check_added_content` warns about internal host or tool names only on lines a PR adds, like `scan-aws-identifiers`, so names already on `main` don't warn on every PR. - With `--base-ref`, `check_history` compares each skill with its merge-base copy: no removed folders, no reuse of a path deleted in `main`'s first-parent history (`git log --first-parent --no-renames --diff-filter=D`, skipped with a warning on a shallow clone), a higher version whenever `content_hash` of the published files changes, one step at a time (next patch, minor or major), and `1.0.0` for a new skill. No folder is ever deleted, not even in an emergency: deprecate it instead. The hash function is pinned by the golden fixture under `content_hash` in `conformance-cases.json`, so any other implementation can prove it computes the same hash. -- With or without a base, a deprecated skill can't stay in any `connectors/*/connector.yaml` `skills:` list (`connector_skill_references`). Every `connectors/*/connector.yaml` and `connectors/releases.yaml` is read with `skill_checks.read_yaml`, the reader the frontmatter uses, so a file that isn't UTF-8 or breaks a YAML rule is an error; the cases are in `skill-rules/yaml-conformance-cases.json`, which `tests/test_skill_checks.py` runs. A `connector.yaml` that isn't a mapping or has a `skills:` value that isn't a list of skill names gets a warning; the names it does list are still checked. +- With or without a base, a deprecated skill can't stay in any `connectors/*/connector.yaml` `skills:` list (`connector_skill_references`). Every `connectors/*/connector.yaml` and `connectors/releases.yaml` is read with `skill_checks.read_yaml`, the reader the frontmatter uses, so a file that isn't UTF-8 or breaks a YAML rule is an error; the cases are in `skill-rules/yaml-conformance-cases.json`, which `tests/test_skill_checks.py` runs. A `connector.yaml` that has a `skills:` value that isn't a list of skill names gets a warning; the names it does list are still checked. One that isn't a mapping is an error from the connector version rules, since it has no version. +- `connector_checks.check_history` compares each `connectors//` folder (not `connectors/schema/`) with its merge-base copy, as the catalog publisher compares a connector with its published version, and reads `connector.yaml` as the publisher does (`skill_checks.plain_scalar_value`); without a base only `check_connector` runs. The rules are under "Connector versions" in CONTRIBUTING.md. - Secrets in test fixtures are built at runtime (`"AKIA" + ...`) or stored as base64 in the conformance cases, so no file in the repository contains a literal key or invisible character. Internal names in tests are made up (`corp.example`, `Rivendell`), never real hosts or tools. - `conformance-cases.json` holds language-neutral cases for these rules. Another tool that validates or publishes these skills should give the same `expect` result on the cases for the rules it implements; some rules here are stricter than any other tool's. The script runs every case before checking any skill and exits 2 on a mismatch. A rule change needs a matching case change in the same PR. - Every rule applies to every skill; nothing is exempted. A rule that existing skills don't meet yet starts as a warning and becomes an error after a cleanup PR brings every skill into line: a missing `metadata.summary`, and dimension values outside the vocabulary. A missing `metadata.aws-devops-agent-skills.agent-types` stays a warning. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 67c004faf..7a21b8113 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,17 @@ Identity and versions, compared with the base branch: - A new skill starts at `1.0.0`. - When the version goes up, add a `CHANGELOG.md` entry for it. A bump without one is a warning. -Connectors: a skill that a connector recommends, in the `skills:` list of a `connectors//connector.yaml`, can't be deprecated unless the same pull request removes it from that list. Each `connector.yaml`, at most 32 KiB, and `connectors/releases.yaml` must be UTF-8 YAML that every parser reads the same way, by the frontmatter's rules, except that a flow collection on one line, a `---` line at the start, `---` inside a value, and a tab inside a quoted value, block scalar or comment are allowed. A flow collection can't span lines, and an unquoted entry in one can't start with `?` or `:`, hold a `?`, or be a lone `-` that no `:` follows. [`yaml-conformance-cases.json`](.github/scripts/skill-rules/yaml-conformance-cases.json) has a case for each rule. A `connector.yaml` that isn't a mapping, or whose `skills:` isn't a list of skill names, is a warning. +Connectors: a skill that a connector recommends, in the `skills:` list of a `connectors//connector.yaml`, can't be deprecated unless the same pull request removes it from that list, which, like any change to a connector's files, needs a higher connector `version`. Each `connector.yaml`, at most 32 KiB, and `connectors/releases.yaml` must be UTF-8 YAML that every parser reads the same way, by the frontmatter's rules, except that a flow collection on one line, a `---` line at the start, `---` inside a value, and a tab inside a quoted value, block scalar or comment are allowed. A flow collection can't span lines, and an unquoted entry in one can't start with `?` or `:`, hold a `?`, or be a lone `-` that no `:` follows. [`yaml-conformance-cases.json`](.github/scripts/skill-rules/yaml-conformance-cases.json) has a case for each rule. A `connector.yaml` that isn't a mapping is an error, because it has no version to check; one whose `skills:` isn't a list of skill names is a warning. + +Connector versions, compared with the base branch as the catalog publisher compares them with the published version ([`connector_checks.py`](.github/scripts/connector_checks.py)): + +- A connector folder is never removed: to retire a connector, set `deprecated: true` in its `connector.yaml` and raise its version. Its `name` and `type` never change. +- `version` is `MAJOR.MINOR.PATCH`, such as `1.0.0`. A new connector starts at `1.0.0`; another version is a warning. +- Every file in a connector folder is published, so a change to any of them, the logo, the README and a comment in `connector.yaml` included, needs a higher `version`. The version never goes down, and going up more than one step (from `1.0.0` to anything but `1.0.1`, `1.1.0` or `2.0.0`) is a warning. +- A change that can break a connection customers already made needs the next major version, such as `2.0.0` after `1.4.2`: changing or removing an `mcp.auth` option (any field counts, a header `label` included), removing an endpoint or changing its URL, in `mcp.endpoint` or `mcp.endpoints`, or adding an `mcp.config` field. Adding an auth option or an endpoint needs only a higher minor or patch version. +- `connector.yaml` is compared by what it means, as the catalog publisher reads it, so comments and quoting don't count toward a major change, but types do: `pkce: "true"` is a change from `pkce: true`, while `endpoint: https://x.example` and `endpoint: "https://x.example"` are the same. + +[`connector-conformance-cases.json`](.github/scripts/skill-rules/connector-conformance-cases.json) has a case for each of these rules. Like the skill rules, they apply to every connector on every pull request; a warning can only come from a connector the pull request changes. The check is exact only for the base commit it ran on. Two pull requests that each pass on an older base can break the rules together, for example by both bumping a skill to the same version, so a later push to `main` can still fail. [`validate-skills-main.yml`](.github/workflows/validate-skills-main.yml) runs the check again after every push to `main` and reports such a break. @@ -124,7 +134,7 @@ git fetch upstream main python3 .github/scripts/validate_skills.py --base-ref upstream/main ``` -Before it checks any skill, the script runs the cases in [`conformance-cases.json`](.github/scripts/skill-rules/conformance-cases.json) through the same rules. When you change a rule, add or update a case for it in the same pull request; the YAML rules also have cases in [`yaml-conformance-cases.json`](.github/scripts/skill-rules/yaml-conformance-cases.json), which the unit tests run. `python3 .github/scripts/validate_skills.py --self-check-only` runs only the cases, and `python3 -m unittest discover -s tests -v` runs the check's unit tests. +Before it checks any skill, the script runs the cases in [`conformance-cases.json`](.github/scripts/skill-rules/conformance-cases.json) through the same rules. When you change a rule, add or update a case for it in the same pull request; the YAML rules also have cases in [`yaml-conformance-cases.json`](.github/scripts/skill-rules/yaml-conformance-cases.json), which the unit tests run, and the connector version rules in [`connector-conformance-cases.json`](.github/scripts/skill-rules/connector-conformance-cases.json), which the script runs with the skill cases. `python3 .github/scripts/validate_skills.py --self-check-only` runs only the cases, and `python3 -m unittest discover -s tests -v` runs the check's unit tests. #### Test Your Skill @@ -587,10 +597,13 @@ A partner whose connector merged before the terms existed accepts them, and adds The box names the version of the terms, the date at the end of `TERMS.md`, so each pull request records the version its partner accepted. One acceptance covers -every later contribution from that partner, connector and skills alike, and a -pull request that removes a connector needs no `contact_email`. A connector is a -definition, not a tool, so no other checklist item applies unless the pull request -also changes a skill. +every later contribution from that partner, connector and skills alike. A +connector is a definition, not a tool, so no other checklist item applies unless +the pull request also changes a skill. + +The `validate-skills` check holds every connector to the connector versions in +[Skill Publishing Rules](#skill-publishing-rules): a change to any of its files +needs a higher `version`, and a connector is never removed, only deprecated. **Revising the terms (maintainers).** AWS Legal approves the new text and updates its record of the terms. One pull request then updates `TERMS.md`, its diff --git a/tests/test_connector_checks.py b/tests/test_connector_checks.py new file mode 100644 index 000000000..020d05981 --- /dev/null +++ b/tests/test_connector_checks.py @@ -0,0 +1,284 @@ +"""Unit tests for .github/scripts/connector_checks.py. + +Each test builds connector folders in memory and runs the rules on them, so no +Git repository is needed. Run with: + + python3 -m unittest discover -s tests -v +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / ".github" / "scripts")) + +import connector_checks as cc # noqa: E402 +import skill_checks as sc # noqa: E402 + +CASES_FILE = ROOT / ".github" / "scripts" / "skill-rules" / cc.CONFORMANCE_FILE + +CONNECTOR = """\ +schema_version: 1 +name: acme +version: 1.0.0 +type: mcp +mcp: + endpoints: + - { label: EU, endpoint: "https://eu.acme.example/mcp" } + - { label: US, endpoint: "https://us.acme.example/mcp" } + auth: + - type: api_key + headers: + - { header: Authorization, label: API key, prefix: "Bearer " } + - type: oauth_3lo + authorization_url: "https://login.acme.example/authorize" + exchange_url: "https://login.acme.example/token" + pkce: true +""" +VERSION_LINE = 3 +API_KEY_OPTION = ' - type: api_key\n headers:\n - { header: Authorization, label: API key, prefix: "Bearer " }\n' + + +def tree(files: dict[str, bytes]) -> sc.SkillTree: + return sc.SkillTree("acme", {path: sc.Entry(path, "100644", data) for path, data in files.items()}) + + +def folder(version: str = "1.0.0", text: str = CONNECTOR, **files: bytes) -> sc.SkillTree: + """A connector folder whose connector.yaml is `text` at `version`.""" + assert "version: 1.0.0\n" in text + connector_yaml = text.replace("version: 1.0.0\n", f"version: {version}\n", 1).encode("utf-8") + return tree({"connector.yaml": connector_yaml, "logo.svg": b"\n", "README.md": b"# Acme\n", **files}) + + +def changed(version: str, old: str, new: str) -> sc.SkillTree: + assert old in CONNECTOR, old + return folder(version, CONNECTOR.replace(old, new)) + + +def messages(findings, severity: str = "error") -> list[str]: + return [f.message for f in findings if f.severity == severity] + + +class ConformanceTests(unittest.TestCase): + def test_repository_cases_give_their_expected_results(self): + self.assertEqual(cc.run_conformance(sc.load_cases(CASES_FILE)), []) + + +class ReadDocumentTests(unittest.TestCase): + def test_values_are_typed_as_the_catalog_publisher_types_them(self): + text = ( + 'a: true\nb: false\nc: null\nd: ~\ne:\nf: 3\ng: -0.25\nh: "3"\ni: 1.0.0\nj: [x, 2, {k: true}]\n' + "l: |\n yes\n" + ) + expected = { + "a": True, "b": False, "c": None, "d": None, "e": None, "f": 3, "g": -0.25, "h": "3", + "i": "1.0.0", "j": ["x", 2, {"k": True}], "l": "yes\n", + } + document = cc.read_document(text.encode("utf-8")) + self.assertTrue(cc.same(document.value, expected), document.value) + self.assertEqual(document.lines["a"], 1) + self.assertEqual(document.lines["l"], 11) + + def test_yaml_that_breaks_a_rule_is_unreadable_unless_lenient(self): + text = b"name: acme\nversion: 1.0.0\nreleased: 2026-11-10\n" + self.assertIsNone(cc.read_document(text)) + lenient = cc.read_document(text, strict=False) + self.assertEqual(lenient.get("version"), "1.0.0") + for unreadable in (b"name: [acme\n", b"name: \xff\n", None): + with self.subTest(unreadable=unreadable): + self.assertIsNone(cc.read_document(unreadable, strict=False)) + + def test_values_compare_strictly(self): + self.assertFalse(cc.same(True, 1)) + self.assertFalse(cc.same(1, "1")) + self.assertFalse(cc.same(None, cc.ABSENT)) + self.assertFalse(cc.same([1, 2], [2, 1])) + self.assertTrue(cc.same({"a": 1, "b": [None]}, {"b": [None], "a": 1})) + self.assertTrue(cc.same(cc.ABSENT, cc.ABSENT)) + + +class CheckConnectorTests(unittest.TestCase): + def test_a_valid_connector_passes(self): + self.assertEqual(cc.check_connector("acme", folder()), []) + + def test_connector_yaml_is_required(self): + findings = cc.check_connector("acme", tree({"README.md": b"# Acme\n"})) + self.assertEqual(messages(findings), ["connectors/acme/ has no connector.yaml; every connector folder needs one"]) + + def test_the_version_must_be_major_minor_patch_text(self): + for version in ('"1.1"', '"v1.0.0"', '"1.0.0-beta"', '"01.0.0"', '"12345678901.0.0"', "1"): + with self.subTest(version=version): + findings = cc.check_connector("acme", folder(version)) + self.assertEqual(len(messages(findings)), 1, findings) + expected = f'"version" is {version}; it must be MAJOR.MINOR.PATCH' + self.assertTrue(findings[0].message.startswith(expected), findings[0].message) + self.assertEqual(findings[0].line, VERSION_LINE) + + def test_a_missing_version_is_required(self): + findings = cc.check_connector("acme", tree({"connector.yaml": b"name: acme\n"})) + self.assertEqual(messages(findings), ['"version" is required, such as version: 1.0.0']) + self.assertIsNone(findings[0].line) + + def test_a_document_that_is_not_a_mapping_is_an_error(self): + # An empty file, or one of only comments, holds no document, which isn't a mapping either. + for text in (b"- acme\n", b"", b"# comment\n"): + with self.subTest(text=text): + findings = cc.check_connector("acme", tree({"connector.yaml": text})) + self.assertEqual(messages(findings), ["connector.yaml must be a mapping of keys to values, such as name: acme"]) + self.assertEqual(findings[0].line, 1) + + def test_yaml_problems_are_left_to_the_yaml_check(self): + self.assertEqual(cc.check_connector("acme", tree({"connector.yaml": b"name: [acme\n"})), []) + + +class HistoryTests(unittest.TestCase): + def test_an_unchanged_connector_passes(self): + self.assertEqual(cc.check_history("acme", folder(), folder()), []) + + def test_a_removed_folder_is_an_error(self): + self.assertIn("set deprecated: true", messages(cc.check_history("acme", folder(), None))[0]) + + def test_any_changed_file_needs_a_higher_version(self): + for head in ( + folder(**{"logo.svg": b"\n"}), + folder(**{"README.md": b"# Acme Cloud\n"}), + folder(**{"logo-dark.svg": b"\n"}), + folder(text="# A comment.\n" + CONNECTOR), + ): + with self.subTest(files=sorted(head.entries)): + findings = cc.check_history("acme", folder(), head) + self.assertEqual(len(messages(findings)), 1, findings) + self.assertIn("so version must go up from 1.0.0", findings[0].message) + version_line = head.entries["connector.yaml"].data.decode().splitlines().index("version: 1.0.0") + 1 + self.assertEqual((findings[0].path, findings[0].line), ("connector.yaml", version_line)) + + def test_a_removed_file_needs_a_higher_version_too(self): + head = folder() + del head.entries["README.md"] + self.assertIn("so version must go up", messages(cc.check_history("acme", folder(), head))[0]) + + def test_the_version_never_goes_down(self): + findings = cc.check_history("acme", folder("1.10.0"), folder("1.9.0")) + self.assertEqual(messages(findings), ["version went down from 1.10.0 to 1.9.0; it must never go down"]) + + def test_name_and_type_never_change(self): + head = folder("2.0.0", CONNECTOR.replace("name: acme", "name: acme-cloud").replace("type: mcp", "type: rest")) + findings = cc.check_history("acme", folder(), head) + self.assertEqual( + messages(findings), + [ + '"name" changed from "acme" to "acme-cloud"; a connector\'s name never changes, so set it back', + '"type" changed from "mcp" to "rest"; a connector\'s type never changes, so set it back', + ], + ) + self.assertEqual([f.line for f in findings], [2, 4]) + + def test_breaking_changes_need_a_new_major_version(self): + api_key_changed = 'the "api_key" option in mcp.auth was removed or changed' + cases = ( + (api_key_changed, "label: API key", "label: Team API key"), + (api_key_changed, API_KEY_OPTION, ""), + ('the "oauth_3lo" option in mcp.auth was removed or changed', "pkce: true", 'pkce: "true"'), + ( + 'the mcp.endpoints entry "US" was removed or its URL changed', + ' - { label: US, endpoint: "https://us.acme.example/mcp" }\n', + "", + ), + ( + 'the mcp.endpoints entry "EU" was removed or its URL changed', + "https://eu.acme.example/mcp", + "https://eu.acme.example/v2/mcp", + ), + ( + 'the mcp.config field "team" was added', + " auth:\n", + ' config:\n - { name: team, label: Team, pattern: "^[a-z]{1,8}$" }\n auth:\n', + ), + ) + for reason, old, new in cases: + with self.subTest(reason=reason, old=old): + for version in ("1.0.1", "1.1.0"): + findings = cc.check_history("acme", folder(), changed(version, old, new)) + self.assertEqual(len(messages(findings)), 1, findings) + self.assertTrue( + findings[0].message.startswith(f"{reason}, which needs a new major version: 2.0.0, not {version}"), + findings[0].message, + ) + self.assertEqual(findings[0].line, VERSION_LINE) + self.assertEqual(cc.check_history("acme", folder(), changed("2.0.0", old, new)), []) + + def test_a_single_endpoint_url_change_needs_a_new_major_version(self): + single = CONNECTOR.replace( + ' endpoints:\n - { label: EU, endpoint: "https://eu.acme.example/mcp" }\n' + ' - { label: US, endpoint: "https://us.acme.example/mcp" }\n', + ' endpoint: "https://mcp.acme.example/mcp"\n', + ) + base = folder("1.0.0", single) + head = folder("1.1.0", single.replace("mcp.acme.example", "api.acme.example")) + self.assertEqual( + [m.split(",")[0] for m in messages(cc.check_history("acme", base, head))], + ["mcp.endpoint was removed or its URL changed"], + ) + + def test_changes_that_keep_connections_working_need_only_a_higher_version(self): + cases = { + "an added auth option": ( + " auth:\n", + ' auth:\n - type: oauth_client_credentials\n exchange_url: "https://login.acme.example/token"\n', + ), + "an added endpoint": ( + ' auth:\n', + ' - { label: AP, endpoint: "https://ap.acme.example/mcp" }\n auth:\n', + ), + "the same value, quoted differently": ( + '"https://login.acme.example/token"', + "https://login.acme.example/token", + ), + } + for label, (old, new) in cases.items(): + with self.subTest(label): + self.assertEqual(cc.check_history("acme", folder(), changed("1.0.1", old, new)), []) + reordered = CONNECTOR.replace(API_KEY_OPTION, "") + API_KEY_OPTION + self.assertEqual(cc.check_history("acme", folder(), folder("1.0.1", reordered)), []) + + def test_a_config_field_that_is_removed_or_kept_needs_no_major_version(self): + with_field = CONNECTOR.replace( + " auth:\n", ' config:\n - { name: team, label: Team, pattern: "^[a-z]{1,8}$" }\n auth:\n' + ) + self.assertEqual(cc.check_history("acme", folder("1.0.0", with_field), folder("1.0.1", with_field)), []) + self.assertEqual(cc.check_history("acme", folder("1.0.0", with_field), folder("1.1.0")), []) + + def test_a_new_connector_starts_at_1_0_0_or_warns(self): + self.assertEqual(cc.check_history("acme", None, folder()), []) + findings = cc.check_history("acme", None, folder("2.0.0")) + self.assertEqual(messages(findings, "warning"), ["a new connector starts at version 1.0.0, not 2.0.0"]) + self.assertEqual(messages(findings), []) + + def test_a_version_that_skips_a_step_warns(self): + findings = cc.check_history("acme", folder("1.0.0"), folder("1.2.0")) + self.assertEqual(messages(findings), []) + self.assertEqual( + messages(findings, "warning"), + ["version should go up only one step from 1.0.0: to 1.0.1, 1.1.0 or 2.0.0, not 1.2.0"], + ) + + def test_an_unreadable_base_version_skips_the_rules_with_a_warning(self): + findings = cc.check_history("acme", folder('"1.0"'), folder("1.0.0", **{"logo.svg": b"\n"})) + self.assertEqual(messages(findings), []) + self.assertIn("the version rules were not checked", messages(findings, "warning")[0]) + + def test_a_base_copy_that_breaks_a_newer_yaml_rule_is_still_compared(self): + base = folder("1.0.0", CONNECTOR + "released: 2026-11-10\n") + head = folder("1.0.0", CONNECTOR + 'released: "2026-11-10"\n') + self.assertIn("so version must go up", messages(cc.check_history("acme", base, head))[0]) + + def test_an_unreadable_head_is_left_to_the_other_rules(self): + self.assertEqual(cc.check_history("acme", folder(), tree({"connector.yaml": b"name: [acme\n"})), []) + self.assertEqual(cc.check_history("acme", folder(), folder('"1.1"')), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_validate_skills.py b/tests/test_validate_skills.py index 6a6b4e6cb..ec7e76969 100644 --- a/tests/test_validate_skills.py +++ b/tests/test_validate_skills.py @@ -353,7 +353,7 @@ def test_removed_skill_folder_fails(self): def test_a_deprecated_skill_cannot_stay_recommended_by_a_connector(self): self.write("skills/alpha/SKILL.md", skill_md("alpha")) - self.write("connectors/acme/connector.yaml", "name: acme\nskills: [alpha, beta]\n") + self.write("connectors/acme/connector.yaml", "name: acme\nversion: 1.0.0\nskills: [alpha, beta]\n") base = self.commit("base") deprecated = skill_md("alpha", version="1.0.1", extra=' deprecated: "true"\n') self.write("skills/alpha/SKILL.md", deprecated) @@ -362,7 +362,8 @@ def test_a_deprecated_skill_cannot_stay_recommended_by_a_connector(self): self.assertEqual(code, 1, out) line = deprecated.splitlines().index(' deprecated: "true"') + 1 self.assertIn(f"FAIL skills/alpha/SKILL.md:{line}: the skill is deprecated, but connectors/acme/connector.yaml still recommends it", out) - self.write("connectors/acme/connector.yaml", "name: acme\nskills: [beta]\n") + # Like any change to a connector's files, dropping a skill from its list needs a higher version. + self.write("connectors/acme/connector.yaml", "name: acme\nversion: 1.0.1\nskills: [beta]\n") self.commit("drop alpha from the connector") code, out = self.run_check("--base-ref", base) self.assertEqual(code, 0, out) @@ -375,8 +376,12 @@ def test_a_connector_yaml_that_breaks_the_yaml_rules_fails_and_an_odd_shape_warn "skills: " + "[" * 2000 + "]" * 2000 + "\n": "FAIL connectors/acme/connector.yaml:1: connector.yaml is nested too deeply", "name: acme\nreleased: 2024-13-45\nskills: [alpha]\n": "FAIL connectors/acme/connector.yaml:2: connector.yaml has the unquoted value 2024-13-45", "port: !!int abc\nskills: [alpha]\n": "FAIL connectors/acme/connector.yaml:1: connector.yaml uses tags", - "name: acme\nskills: alpha\n": "WARN connectors/acme/connector.yaml: has a skills: value that isn't a list", - "- alpha\n": "WARN connectors/acme/connector.yaml: isn't a YAML mapping", + "name: acme\nversion: 1.0.0\nskills: alpha\n": "WARN connectors/acme/connector.yaml: has a skills: value that isn't a list", + # Not a mapping, so it has no version either, which the connector version rules need. + "- alpha\n": "FAIL connectors/acme/connector.yaml:1: connector.yaml must be a mapping of keys to values", + # An empty file, or one of only comments, holds no document, which isn't a mapping either. + "": "FAIL connectors/acme/connector.yaml:1: connector.yaml must be a mapping of keys to values", + "# comment\n": "FAIL connectors/acme/connector.yaml:1: connector.yaml must be a mapping of keys to values", } for text, expected in cases.items(): with self.subTest(text=text[:24]): @@ -385,6 +390,7 @@ def test_a_connector_yaml_that_breaks_the_yaml_rules_fails_and_an_odd_shape_warn code, out = self.run_check("--base-ref", base) self.assertEqual(code, 1 if expected.startswith("FAIL") else 0, out) self.assertIn(expected, out) + self.assertNotIn("isn't a YAML mapping", out) def test_a_connector_yaml_over_32_kib_fails(self): self.write("skills/alpha/SKILL.md", skill_md("alpha")) @@ -418,8 +424,8 @@ def test_a_releases_file_that_breaks_the_yaml_rules_fails(self): def test_odd_connector_lists_still_find_the_skill(self): cases = { - "name: acme\nskills: [alpha, null]\n": "has a skills: entry that isn't a skill name", - "name: acme\nskills:\n - 'alpha'\n": None, + "name: acme\nversion: 1.0.0\nskills: [alpha, null]\n": "has a skills: entry that isn't a skill name", + "name: acme\nversion: 1.0.0\nskills:\n - 'alpha'\n": None, } for text, warning in cases.items(): with self.subTest(text=text[:24]): @@ -557,6 +563,74 @@ def test_merge_base_not_base_tip_is_the_comparison(self): self.assertEqual(self.git("merge-base", "main", "feature"), fork_point) +def connector_yaml(version: str = "1.0.0", name: str = "acme") -> str: + return ( + f"schema_version: 1\nname: {name}\nversion: {version}\ntype: mcp\nmcp:\n" + ' endpoint: "https://mcp.acme.example/mcp"\n' + " auth:\n - type: api_key\n headers:\n" + ' - { header: Authorization, label: API key, prefix: "Bearer " }\n' + ) + + +class ConnectorVersionTests(RepoTestCase): + def setUp(self): + super().setUp() + self.write("skills/alpha/SKILL.md", skill_md("alpha")) + self.write("connectors/acme/connector.yaml", connector_yaml()) + self.write("connectors/acme/logo.svg", "\n") + self.write("connectors/releases.yaml", "schema_version: 1\nconnectors: []\n") + self.write("connectors/schema/v1/published-files.json", '{"files": []}\n') + self.base = self.commit("base") + + def test_a_changed_file_at_the_same_version_fails_until_the_version_goes_up(self): + self.write("connectors/acme/logo.svg", "\n") + self.commit("a new logo") + code, out = self.run_check("--base-ref", self.base, env={"GITHUB_ACTIONS": "true"}) + self.assertEqual(code, 1, out) + self.assertIn( + "FAIL connectors/acme/connector.yaml:3: files in connectors/acme/ changed, so version must go up from 1.0.0", + out, + ) + self.assertIn("::error file=connectors/acme/connector.yaml,line=3,title=Skill publishing rules::", out) + self.assertIn("1 skill(s) checked: 0 with errors. 1 other problem(s) in the repository.", out) + self.write("connectors/acme/connector.yaml", connector_yaml("1.0.1")) + self.commit("bump the version") + code, out = self.run_check("--base-ref", self.base) + self.assertEqual(code, 0, out) + + def test_a_removed_connector_folder_fails(self): + self.git("rm", "-q", "-r", "connectors/acme") + self.commit("remove acme") + code, out = self.run_check("--base-ref", self.base) + self.assertEqual(code, 1, out) + self.assertIn("FAIL connectors/acme: the connector folder connectors/acme was removed", out) + + def test_the_schema_folder_and_shared_files_are_not_connectors(self): + self.write("connectors/schema/v1/published-files.json", '{"files": [], "note": "changed"}\n') + self.write("connectors/README.md", "# Connectors\n") + self.commit("change shared files") + code, out = self.run_check("--base-ref", self.base) + self.assertEqual(code, 0, out) + self.assertNotIn("connectors/schema", out) + + def test_a_new_connector_not_at_1_0_0_warns(self): + self.write("connectors/bravo/connector.yaml", connector_yaml("2.0.0", name="bravo")) + self.commit("add bravo") + code, out = self.run_check("--base-ref", self.base) + self.assertEqual(code, 0, out) + self.assertIn("WARN connectors/bravo/connector.yaml:3: a new connector starts at version 1.0.0, not 2.0.0", out) + code, out = self.run_check() + self.assertEqual(code, 0, out) + self.assertNotIn("connectors/bravo", out) + + def test_without_a_base_only_the_version_itself_is_checked(self): + self.write("connectors/acme/connector.yaml", connector_yaml('"1.1"')) + self.commit("a two-part version") + code, out = self.run_check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL connectors/acme/connector.yaml:3: "version" is "1.1"; it must be MAJOR.MINOR.PATCH', out) + + class SelfCheckTests(RepoTestCase): def test_repository_cases_pass(self): code, out = self.run_check("--self-check-only") @@ -576,6 +650,17 @@ def test_wrong_case_stops_the_check(self): self.assertIn("case valid-minimal: expected an error", out) self.assertNotIn("skill(s) checked", out) + def test_a_wrong_connector_case_stops_the_check(self): + rules_dir = self.rules_copy() + path = rules_dir / "connector-conformance-cases.json" + document = json.loads(path.read_text(encoding="utf-8")) + unchanged = next(case for case in document["cases"] if case["id"] == "unchanged") + unchanged["expect"], unchanged["error"] = "fail", "anything" + path.write_text(json.dumps(document), encoding="utf-8") + code, out = self.run_check("--self-check-only", rules_dir=rules_dir) + self.assertEqual(code, 2, out) + self.assertIn("connector case unchanged: expected an error containing 'anything'", out) + def test_changed_hash_function_stops_the_check(self): rules_dir = self.rules_copy() path = rules_dir / validate_skills.CONFORMANCE_FILE From a7c178d6a490d14d915f5e85f0b6057469206d1c Mon Sep 17 00:00:00 2001 From: miscreantmoogly Date: Wed, 11 Nov 2026 05:26:44 -0700 Subject: [PATCH 3/6] fix(ci): read skill-rules/published-files.json from the merge base published-files.json decides which files in a skill are published, and so which changes need a version bump. The check read it from the pull request's own copy, so a pull request that widened the list was compared on its new list: a file it left out of the published set and changed in the same pull request needed no bump. Read the list from the merge base, as the check already reads the extension allowlist in skills/.gitignore, so a change to it takes effect only once it has merged. Both are read in the same step, through one helper that reads a file at a revision. The conformance cases still run on the pull request's own rules; only its skills are compared on the base branch's list. A malformed list on the base branch stops the check with exit 2, as malformed rule data does. skill_checks.published_excludes holds the list's validation, which load_rules used inline. Code-owner review of .github/ remains what stops a pull request that edits the check itself. --- .claude/CLAUDE.md | 2 +- .github/scripts/skill_checks.py | 17 +++++++++--- .github/scripts/validate_skills.py | 39 ++++++++++++++++++++++++--- .kiro/steering/project-conventions.md | 2 +- CONTRIBUTING.md | 2 +- tests/test_validate_skills.py | 38 ++++++++++++++++++++++++++ 6 files changed, 89 insertions(+), 11 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index c84f10d53..427457a25 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -95,7 +95,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. -- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. +- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. - The extension allowlist is not duplicated: it is parsed from the `!*.` lines of `skills/.gitignore`, read from the merge base, so a new extension takes effect only once its `.gitignore` change has merged. - Like the repository's other checks, a PR runs its own copy of the script, rules and tests, so it could edit them to pass. Code-owner review of `/.github/`, `/tests/` and `/.gitattributes` (`.github/CODEOWNERS`) is the control; don't describe any in-script measure as stopping a determined PR. The post-merge run on `main` catches pull requests that break the rules together, not a pull request that edits the check. - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. diff --git a/.github/scripts/skill_checks.py b/.github/scripts/skill_checks.py index d61765007..e833862b8 100644 --- a/.github/scripts/skill_checks.py +++ b/.github/scripts/skill_checks.py @@ -322,9 +322,7 @@ def load_rules(rules_dir: Path, gitignore_text: str) -> Rules: published = _read_json(rules_dir / PUBLISHED_FILES_FILE) internal = _read_json(rules_dir / INTERNAL_NAMES_FILE) try: - excludes = _string_list(published["exclude"], f"{PUBLISHED_FILES_FILE} exclude") - if not excludes or not all("/" not in e.rstrip("/") for e in excludes): - raise TypeError(f"{PUBLISHED_FILES_FILE}: exclude must list root-level names, folders ending in /") + excludes = published_excludes(published) values = _string_list(agent_types["values"], f"{AGENT_TYPES_FILE} values") dimensions = vocabulary["values"] aliases = vocabulary["aliases"] @@ -342,13 +340,24 @@ def load_rules(rules_dir: Path, gitignore_text: str) -> Rules: vocabulary=vocabulary_sets, aliases=dict(aliases), allowed_extensions=allowed_extensions_from_gitignore(gitignore_text), - published_excludes=tuple(excludes), + published_excludes=excludes, internal_names=_internal_name_patterns(internal), ) except (KeyError, TypeError, AttributeError) as exc: raise ValueError(f"malformed rule data in {rules_dir}: {exc}") from exc +def published_excludes(document) -> tuple[str, ...]: + """The exclude list of a published-files.json document. + + Raises KeyError, TypeError or AttributeError when the document is malformed. + """ + excludes = _string_list(document["exclude"], f"{PUBLISHED_FILES_FILE} exclude") + if not excludes or not all("/" not in e.rstrip("/") for e in excludes): + raise TypeError(f"{PUBLISHED_FILES_FILE}: exclude must list root-level names, folders ending in /") + return tuple(excludes) + + def _internal_name_patterns(document) -> tuple[re.Pattern[str], ...]: """internal-names.json as patterns: each host with any subdomain, each term as a whole word, ignoring case.""" if not isinstance(document, dict): diff --git a/.github/scripts/validate_skills.py b/.github/scripts/validate_skills.py index 8fbaae293..c2f93c3c7 100644 --- a/.github/scripts/validate_skills.py +++ b/.github/scripts/validate_skills.py @@ -34,6 +34,8 @@ from __future__ import annotations import argparse +import dataclasses +import json import os import subprocess import sys @@ -51,6 +53,8 @@ MAX_CONNECTOR_FILE_BYTES = 32_768 RELEASES_PATH = f"{CONNECTORS_DIR}/releases.yaml" RULES_DIR = Path(__file__).resolve().parent / "skill-rules" +# Read from the merge base, like skills/.gitignore. +PUBLISHED_FILES_PATH = f"{sc.RULES_DIR_DISPLAY}/{sc.PUBLISHED_FILES_FILE}" CONFORMANCE_FILE = "conformance-cases.json" DOCS_HINT = 'See the "Skill Publishing Rules" section of CONTRIBUTING.md.' @@ -100,6 +104,12 @@ def _ls_tree(repo_root: Path, rev: str, *paths: str) -> Iterator[tuple[str, str, yield mode, kind, oid, _decode_path(raw_path) +def _read_file(repo_root: Path, rev: str, path: str) -> bytes | None: + """The bytes of the file at `path` in `rev`, or None when `rev` has no file there.""" + oids = [oid for _, kind, oid, found in _ls_tree(repo_root, rev, path) if kind == "blob" and found == path] + return _read_blobs(repo_root, oids)[0] if oids else None + + def read_skills(repo_root: Path, rev: str) -> tuple[dict[str, sc.SkillTree], list[SkillResult]]: """Every skill folder at `rev`, plus results for entries in skills/ that aren't skill folders.""" trees: dict[str, sc.SkillTree] = {} @@ -173,7 +183,10 @@ def uncommitted_skill_changes(repo_root: Path) -> bool: def read_gitignore(repo_root: Path, rev: str) -> str: - return _git(repo_root, "show", f"{rev}:{SKILLS_DIR}/.gitignore").decode("utf-8") + data = _read_file(repo_root, rev, f"{SKILLS_DIR}/.gitignore") + if data is None: + raise GitError(f"{rev} has no {SKILLS_DIR}/.gitignore") + return data.decode("utf-8") def repository_results(repo_root: Path, rev: str) -> list[SkillResult]: @@ -283,6 +296,18 @@ def connector_results(repo_root: Path, base: str | None, head_ref: str) -> list[ return results +def base_published_excludes(data: bytes) -> tuple[str, ...]: + """The exclude list in the merge base's skill-rules/published-files.json. Raises ValueError when malformed. + + The base branch's list applies, so a change to it takes effect only once it + has merged, as a new extension in skills/.gitignore does. + """ + try: + return sc.published_excludes(json.loads(data.decode("utf-8"))) + except (ValueError, KeyError, TypeError, AttributeError) as exc: + raise ValueError(f"{PUBLISHED_FILES_PATH} on the base branch is malformed: {exc}") from exc + + def connector_skill_references(repo_root: Path, rev: str) -> tuple[dict[str, list[str]], list[SkillResult]]: """Skill name -> the connectors whose connector.yaml lists it under skills: (recommended skills). @@ -534,16 +559,18 @@ def _run(argv: list[str] | None, repo_root: Path | None, rules_dir: Path) -> int repo_root = repo_root or _repo_root() base = None + published = None try: if args.self_check_only: # The cases test the rules, not the repository, so no Git is needed. gitignore = (repo_root / SKILLS_DIR / ".gitignore").read_text(encoding="utf-8") else: - # The extension allowlist is read from the merge base when there is - # one, so a new extension takes effect only after the - # skills/.gitignore change that adds it has merged. + # The extension allowlist and the published set are read from the + # merge base when there is one, so a new extension, or a change to + # what is published, takes effect only after its change has merged. base = merge_base(repo_root, args.base_ref, args.head_ref) if args.base_ref else None gitignore = read_gitignore(repo_root, base or args.head_ref) + published = _read_file(repo_root, base, PUBLISHED_FILES_PATH) if base is not None else None except OSError as exc: print(f"::error title=Skill publishing rules::cannot read {SKILLS_DIR}/.gitignore: {_escape_data(str(exc))}") return 2 @@ -551,6 +578,7 @@ def _run(argv: list[str] | None, repo_root: Path | None, rules_dir: Path) -> int return _could_not_run(exc) try: rules = sc.load_rules(rules_dir, gitignore) + excludes = base_published_excludes(published) if published is not None else None except ValueError as exc: print(f"::error title=Skill publishing rules::{_escape_data(str(exc))}") return 2 @@ -564,6 +592,9 @@ def _run(argv: list[str] | None, repo_root: Path | None, rules_dir: Path) -> int print("Self-check passed.") return 0 + if excludes is not None: + # The cases above test this pull request's rules; its skills are held to the base branch's published set. + rules = dataclasses.replace(rules, published_excludes=excludes) try: if args.head_ref == "HEAD" and uncommitted_skill_changes(repo_root): print(f"note: uncommitted changes under {SKILLS_DIR}/ are not checked; commit them first.\n") diff --git a/.kiro/steering/project-conventions.md b/.kiro/steering/project-conventions.md index 931499bc9..744158a1f 100644 --- a/.kiro/steering/project-conventions.md +++ b/.kiro/steering/project-conventions.md @@ -95,7 +95,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. -- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. +- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. - The extension allowlist is not duplicated: it is parsed from the `!*.` lines of `skills/.gitignore`, read from the merge base, so a new extension takes effect only once its `.gitignore` change has merged. - Like the repository's other checks, a PR runs its own copy of the script, rules and tests, so it could edit them to pass. Code-owner review of `/.github/`, `/tests/` and `/.gitattributes` (`.github/CODEOWNERS`) is the control; don't describe any in-script measure as stopping a determined PR. The post-merge run on `main` catches pull requests that break the rules together, not a pull request that edits the check. - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a21b8113..0e3546014 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,7 +76,7 @@ The frontmatter must be plain YAML that every parser reads the same way. These a A `SKILL.md` body over 500 lines is a warning, as it is in the skill evaluation tool: move reference material into `references/` and link to it. -Published files. Everything in the skill folder is published and installed into customers' Agent Spaces except `evals/`, `.skilleval.yaml` and `CHANGELOG.md` at the skill root ([`published-files.json`](.github/scripts/skill-rules/published-files.json)). `README.md` and `images/` are published too. For the published files: +Published files. Everything in the skill folder is published and installed into customers' Agent Spaces except `evals/`, `.skilleval.yaml` and `CHANGELOG.md` at the skill root ([`published-files.json`](.github/scripts/skill-rules/published-files.json)). `README.md` and `images/` are published too. The check reads that list from the base branch, so a change to it takes effect only after it has merged. For the published files: - At most 100 files, each at most 1 MiB, and their zip at most 4,128,768 bytes (4 MiB less a margin, because zip sizes vary by tool). The check warns past 90% of each of these limits. The `SKILL.md` frontmatter is at most 32 KiB. - Each file's extension must be one that `skills/.gitignore` allows. The check reads that list from the base branch, so a new extension takes effect only after the `skills/.gitignore` change that adds it has merged. diff --git a/tests/test_validate_skills.py b/tests/test_validate_skills.py index ec7e76969..9d0a7d7f5 100644 --- a/tests/test_validate_skills.py +++ b/tests/test_validate_skills.py @@ -631,6 +631,44 @@ def test_without_a_base_only_the_version_itself_is_checked(self): self.assertIn('FAIL connectors/acme/connector.yaml:3: "version" is "1.1"; it must be MAJOR.MINOR.PATCH', out) +class PublishedFilesFromTheBaseTests(RepoTestCase): + """skill-rules/published-files.json is read from the merge base, so a change to it applies once it has merged.""" + + PATH = validate_skills.PUBLISHED_FILES_PATH + + def published_files(self, *exclude: str) -> str: + return json.dumps({"description": "Test copy.", "exclude": list(exclude)}) + + def test_a_name_the_pull_request_leaves_out_of_the_published_set_still_needs_a_bump(self): + self.write(self.PATH, self.published_files("evals/", ".skilleval.yaml", "CHANGELOG.md")) + self.write("skills/alpha/SKILL.md", skill_md("alpha")) + self.write("skills/alpha/NOTES.md", "one\n") + base = self.commit("base") + rules_dir = self.rules_copy() + widened = self.published_files("evals/", ".skilleval.yaml", "CHANGELOG.md", "NOTES.md") + (rules_dir / "published-files.json").write_text(widened, encoding="utf-8") + self.write(self.PATH, widened) + self.write("skills/alpha/NOTES.md", "two\n") + merged = self.commit("leave NOTES.md out of the published set, and change it") + code, out = self.run_check("--base-ref", base, rules_dir=rules_dir) + self.assertEqual(code, 1, out) + self.assertIn("metadata.version must go up from 1.0.0", out) + # Once the list has merged, it applies. + self.write("skills/alpha/NOTES.md", "three\n") + self.commit("change NOTES.md again") + code, out = self.run_check("--base-ref", merged, rules_dir=rules_dir) + self.assertEqual(code, 0, out) + + def test_a_malformed_list_on_the_base_branch_cannot_run(self): + self.write(self.PATH, '{"exclude": "evals/"}') + self.write("skills/alpha/SKILL.md", skill_md("alpha")) + base = self.commit("base") + code, out = self.run_check("--base-ref", base) + self.assertEqual(code, 2, out) + self.assertIn("published-files.json on the base branch is malformed", out) + self.assertNotIn("skill(s) checked", out) + + class SelfCheckTests(RepoTestCase): def test_repository_cases_pass(self): code, out = self.run_check("--self-check-only") From c0669d534a03af4c413a9e4a0cd48c699ff52180 Mon Sep 17 00:00:00 2001 From: miscreantmoogly Date: Tue, 10 Nov 2026 23:32:41 -0700 Subject: [PATCH 4/6] feat(ci): reject placeholder tag values such as "All" A skill that lists "All" in aws-services shows up in catalogs under a service called "All", because nothing between the skill and the filter turns a placeholder back into "every service". The check only warned about it, as it does for any value missing from vocabulary.json. Make All, Any, None, N/A, Other, General and *, compared ignoring case, an error in aws-services and technical-domains. The message tells the author to leave the key out, since a skill without it still appears when no filter is chosen, or, when the list also holds real values, to remove the placeholder from it. vocabulary.json can't approve a placeholder either. load_rules refuses one in values or in aliases, so the check exits 2 instead of letting a pull request approve its own placeholder. Add the dimension-reserved-value conformance case, unit tests for the rule and for load_rules, and describe the rule in CONTRIBUTING.md, vocabulary.json and the project conventions. --- .claude/CLAUDE.md | 2 +- .../skill-rules/conformance-cases.json | 11 ++++++ .github/scripts/skill-rules/vocabulary.json | 2 +- .github/scripts/skill_checks.py | 32 ++++++++++++++++- .kiro/steering/project-conventions.md | 2 +- CONTRIBUTING.md | 2 +- tests/test_skill_checks.py | 35 +++++++++++++++++++ 7 files changed, 81 insertions(+), 5 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 427457a25..1e797723e 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -95,7 +95,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. -- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. +- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn, a placeholder such as `All`, `None` or `*` is an error in any case, and the file can't list one), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. - The extension allowlist is not duplicated: it is parsed from the `!*.` lines of `skills/.gitignore`, read from the merge base, so a new extension takes effect only once its `.gitignore` change has merged. - Like the repository's other checks, a PR runs its own copy of the script, rules and tests, so it could edit them to pass. Code-owner review of `/.github/`, `/tests/` and `/.gitattributes` (`.github/CODEOWNERS`) is the control; don't describe any in-script measure as stopping a determined PR. The post-merge run on `main` catches pull requests that break the rules together, not a pull request that edits the check. - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. diff --git a/.github/scripts/skill-rules/conformance-cases.json b/.github/scripts/skill-rules/conformance-cases.json index 329ec505a..bf1c746a4 100644 --- a/.github/scripts/skill-rules/conformance-cases.json +++ b/.github/scripts/skill-rules/conformance-cases.json @@ -355,6 +355,17 @@ } } }, + { + "id": "dimension-reserved-value", + "name": "demo", + "expect": "fail", + "error": "is a placeholder", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\n aws-devops-agent-skills.aws-services: \"All\"\n aws-devops-agent-skills.technical-domains: \"none\"\n---\n\n# Demo\n" + } + } + }, { "id": "dimension-unknown", "name": "demo", diff --git a/.github/scripts/skill-rules/vocabulary.json b/.github/scripts/skill-rules/vocabulary.json index 962e6882d..30c2f68bd 100644 --- a/.github/scripts/skill-rules/vocabulary.json +++ b/.github/scripts/skill-rules/vocabulary.json @@ -1,5 +1,5 @@ { - "description": "Approved values for the metadata.aws-devops-agent-skills.* dimensions, which skill catalogs such as GitHub Pages show as filters. A value not listed here is a warning, not an error. aliases maps a value in use today to the approved spelling the check suggests instead. agent-types is absent on purpose: it sets which agents load the skill, so its values are the API values in agent-types.json and an unknown one is an error.", + "description": "Approved values for the metadata.aws-devops-agent-skills.* dimensions, which skill catalogs such as GitHub Pages show as filters. A value not listed here is a warning, not an error. A placeholder such as All, Any, None, N/A, Other, General or *, in any case, is always an error, so neither values nor aliases may list one; the check refuses to run if they do. aliases maps a value in use today to the approved spelling the check suggests instead. agent-types is absent on purpose: it sets which agents load the skill, so its values are the API values in agent-types.json and an unknown one is an error.", "values": { "aws-services": [ "AWS Backup", diff --git a/.github/scripts/skill_checks.py b/.github/scripts/skill_checks.py index e833862b8..68a01d294 100644 --- a/.github/scripts/skill_checks.py +++ b/.github/scripts/skill_checks.py @@ -73,6 +73,11 @@ AGENT_TYPES_DIMENSION = "agent-types" AGENT_TYPES_KEY = DIMENSION_PREFIX + AGENT_TYPES_DIMENSION VOCABULARY_DIMENSIONS = tuple(d for d in DIMENSIONS if d != AGENT_TYPES_DIMENSION) +# Placeholders that stand for every value, or for none, rather than naming one. +# Catalogs would show one as a filter option of its own, so a skill can't use +# one in aws-services or technical-domains, and vocabulary.json can't list one. +# Compared ignoring case. +RESERVED_DIMENSION_VALUES = frozenset({"all", "any", "none", "n/a", "other", "general", "*"}) # Loads the skill for every agent, so it can't be combined with another type. GENERIC_AGENT_TYPE = "GENERIC" @@ -335,6 +340,13 @@ def load_rules(rules_dir: Path, gitignore_text: str) -> Rules: vocabulary_sets = { dim: frozenset(_string_list(dimensions[dim], f"{VOCABULARY_FILE} {dim}")) for dim in VOCABULARY_DIMENSIONS } + listed = [value for values in vocabulary_sets.values() for value in values] + [*aliases, *aliases.values()] + reserved = sorted(value for value in listed if is_reserved_dimension_value(value)) + if reserved: + raise TypeError( + f'{VOCABULARY_FILE}: "{reserved[0]}" is a placeholder, which the check refuses as a value, ' + "so it can't be listed" + ) return Rules( agent_types=frozenset(values), vocabulary=vocabulary_sets, @@ -381,6 +393,11 @@ def _string_list(value, label: str) -> list[str]: return value +def is_reserved_dimension_value(value: str) -> bool: + """Whether a dimension value is a placeholder such as All, N/A or *, in any case.""" + return value.casefold() in RESERVED_DIMENSION_VALUES + + def allowed_extensions_from_gitignore(text: str) -> frozenset[str]: """The extensions skills/.gitignore allows, from its `!*.` lines, lowercased.""" extensions = frozenset( @@ -1208,9 +1225,22 @@ def _check_dimensions(metadata: dict, items: dict, rules: Rules, report: _Report if dimension == AGENT_TYPES_DIMENSION: continue # _check_agent_types validates it against agent-types.json. approved = rules.vocabulary.get(dimension, frozenset()) - for part in _split(metadata[key]): + parts = _split(metadata[key]) + # Leaving the key out is the fix only when nothing else would be left in it. + fix = ( + "Remove it from the list" + if any(part and not is_reserved_dimension_value(part) for part in parts) + else f'Leave "metadata.{key}" out instead; a skill without it still appears when no filter is chosen' + ) + for part in parts: if not part: report.error(f'"metadata.{key}" has an empty value between commas', line=line) + elif is_reserved_dimension_value(part): + report.error( + f'"{part}" is a placeholder rather than a value, and catalogs would list it as a filter option ' + f"of its own. {fix}", + line=line, + ) elif part not in approved: hint = ( f'use "{rules.aliases[part]}"' diff --git a/.kiro/steering/project-conventions.md b/.kiro/steering/project-conventions.md index 744158a1f..cc5ade45b 100644 --- a/.kiro/steering/project-conventions.md +++ b/.kiro/steering/project-conventions.md @@ -95,7 +95,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. -- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. +- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn, a placeholder such as `All`, `None` or `*` is an error in any case, and the file can't list one), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. - The extension allowlist is not duplicated: it is parsed from the `!*.` lines of `skills/.gitignore`, read from the merge base, so a new extension takes effect only once its `.gitignore` change has merged. - Like the repository's other checks, a PR runs its own copy of the script, rules and tests, so it could edit them to pass. Code-owner review of `/.github/`, `/tests/` and `/.gitattributes` (`.github/CODEOWNERS`) is the control; don't describe any in-script measure as stopping a determined PR. The post-merge run on `main` catches pull requests that break the rules together, not a pull request that edits the check. - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e3546014..18feefe97 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,7 +70,7 @@ Skills on `main` are published automatically, so a pull request check ([`.github | `metadata.deprecated` | Optional. `"true"` or `"false"`. | | `metadata.tier` | Optional. `"aws-managed"` or `"partner-managed"`: who maintains the skill. | | `metadata.aws-devops-agent-skills.agent-types` | The agents that load the skill, as comma-separated values from [`agent-types.json`](.github/scripts/skill-rules/agent-types.json), such as `"INCIDENT_RCA"`. An unknown value is an error, because this key decides which agents load the skill rather than only how catalogs filter it. `GENERIC` loads the skill for every agent, so it is an error to combine it with another value. Without the key the skill installs as `GENERIC` and loads for every agent, which is a warning. | -| `metadata.aws-devops-agent-skills.*` | Only `agent-types`, `aws-services` and `technical-domains`, each a comma-separated list. Any other key under this prefix, or a misspelling of the prefix, is an error, so a typo can't drop the skill from a filter. For `aws-services` and `technical-domains`, a value not in [`vocabulary.json`](.github/scripts/skill-rules/vocabulary.json) is a warning, with the approved spelling when there is one. | +| `metadata.aws-devops-agent-skills.*` | Only `agent-types`, `aws-services` and `technical-domains`, each a comma-separated list. Any other key under this prefix, or a misspelling of the prefix, is an error, so a typo can't drop the skill from a filter. For `aws-services` and `technical-domains`, a value not in [`vocabulary.json`](.github/scripts/skill-rules/vocabulary.json) is a warning, with the approved spelling when there is one. A placeholder such as `All`, `Any`, `None`, `N/A`, `Other`, `General` or `*`, in any case, is an error, because catalogs would list it as a filter option of its own: leave the key out instead, and the skill still appears when no filter is chosen. | The frontmatter must be plain YAML that every parser reads the same way. These are rejected: tabs; a byte order mark anywhere; line breaks other than newlines (U+0085, U+2028, U+2029, or a carriage return without a line feed); `---` anywhere but the closing line; anchors (`&`), aliases (`*`), tags (`!`), directives (`%`) and document markers; flow collections (`[...]` or `{...}`); keys that aren't plain text, `<<` keys, and duplicate keys, at any depth; nesting more than 64 levels deep; unquoted values that some parsers read as another type, such as `1e3`, `yes`, `y` or `2026-11-08`; in a double-quoted value, an escaped UTF-16 surrogate such as `\uD83D`, or an escaped line break before a blank line; a block scalar indentation indicator, such as `|2`; a block scalar whose first line is indented less than a blank line before it; a `#` right after a closing quote or bracket; a quoted value whose next line isn't indented past its key or list item with spaces; and a block scalar as the last key, unless it is `|-` or `>-`. A top-level key other than the Agent Skills specification's fields and `title` is a warning. diff --git a/tests/test_skill_checks.py b/tests/test_skill_checks.py index ad07ba742..bebfd271c 100644 --- a/tests/test_skill_checks.py +++ b/tests/test_skill_checks.py @@ -488,6 +488,29 @@ def test_empty_dimension_value_is_an_error(self): fm = frontmatter_with(**{"aws-devops-agent-skills.aws-services": '"Amazon EKS, "'}) self.assertOneError(tree(skill_md(fm)), "empty value between commas") + def test_placeholder_values_are_errors_in_any_case(self): + for key in ("aws-devops-agent-skills.aws-services", "aws-devops-agent-skills.technical-domains"): + for value in ("All", "any", "NONE", "N/A", "n/a", "Other", "General", "*"): + with self.subTest(key=key, value=value): + skill = tree(skill_md(frontmatter_with(**{key: f'"{value}"'}))) + error = self.assertOneError(skill, f'"{value}" is a placeholder', line=9) + self.assertIn(f'Leave "metadata.{key}" out instead', error.message) + self.assertEqual(self.warnings(skill), []) + + def test_a_placeholder_beside_real_values_is_removed_from_the_list(self): + fm = frontmatter_with(**{"aws-devops-agent-skills.aws-services": '"Amazon EKS, All"'}) + error = self.assertOneError(tree(skill_md(fm)), '"All" is a placeholder') + self.assertTrue(error.message.endswith("Remove it from the list"), error.message) + + def test_values_that_only_contain_a_placeholder_word_are_not_placeholders(self): + fm = frontmatter_with(**{"aws-devops-agent-skills.technical-domains": '"Operations, All Services, n/a/b"'}) + skill = tree(skill_md(fm)) + self.assertFalse(any("placeholder" in f.message for f in self.check(skill)[0])) + + def test_agent_types_report_a_placeholder_as_an_unknown_agent_type(self): + fm = frontmatter_with(**{"aws-devops-agent-skills.agent-types": '"All"'}) + self.assertOneError(tree(skill_md(fm)), 'unknown value "All"') + def test_unapproved_value_is_a_warning_with_suggestion(self): fm = frontmatter_with(**{"aws-devops-agent-skills.aws-services": '"CloudWatch, Amazon S3"'}) skill = tree(skill_md(fm)) @@ -1167,6 +1190,18 @@ def test_aliases_must_map_strings_to_strings(self): with self.assertRaisesRegex(ValueError, "aliases"): sc.load_rules(self.rules_dir(None, vocabulary), "!*.md\n") + def test_a_placeholder_in_the_vocabulary_fails_closed(self): + cases = { + "All": {"values": {"aws-services": ["Amazon EKS", "All"], "technical-domains": []}, "aliases": {}}, + "n/a": {"values": {"aws-services": [], "technical-domains": ["n/a"]}, "aliases": {}}, + "Any": {"values": {"aws-services": ["Amazon EKS"], "technical-domains": []}, "aliases": {"Any": "Amazon EKS"}}, + "NONE": {"values": {"aws-services": [], "technical-domains": []}, "aliases": {"EKS": "NONE"}}, + } + for value, vocabulary in cases.items(): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, f'"{value}" is a placeholder'): + sc.load_rules(self.rules_dir(None, vocabulary), "!*.md\n") + class ConformanceTests(unittest.TestCase): def write_cases(self, cases) -> Path: From fbf06c7c72d47b0491fe774d29b28ba1b53e34de Mon Sep 17 00:00:00 2001 From: miscreantmoogly Date: Tue, 10 Nov 2026 23:38:54 -0700 Subject: [PATCH 5/6] feat(ci): check that skill authors are GitHub accounts The check only tests that metadata.author has the form of a GitHub login, so a skill can name an account that doesn't exist. Catalogs link the name to its GitHub profile, and that link then 404s, or goes to whoever registers the name later. Add .github/scripts/github_logins.py, which looks a login up with GitHub's user and organization lookup (GET /users/): a 10-second timeout, one retry for a server error or a failed request, one lookup per login per run, and redirects refused, so the token never goes anywhere but api.github.com. It reads the token from GH_TOKEN, then GITHUB_TOKEN, as the GitHub CLI does. validate_skills.py --check-github-logins looks up each name that a new skill lists, or that a changed author line adds. Names already on the base branch aren't looked up again, so an account deleted later can't fail a pull request that leaves the author line alone. A 404 is an error. When GitHub can't answer, because no token is set, the rate limit is reached, the server fails or the request times out, the name is a warning, so an outage never blocks a pull request. After the first lookup GitHub can't answer, the run asks nothing more and every later name is a warning too, so an outage costs one lookup's timeout and retry rather than one per name, which could outlast the job's time limit, and a rate limit isn't hit again. A team is looked up by its organization, since GitHub shows a team only to the organization's members, and a team of a user account is an error. The pull request workflow passes the flag and the job's token in GH_TOKEN, the name the repository's other workflows use, keeping permissions at contents: read. Tests answer with a stub resolver or a fake HTTPS handler, so none reaches GitHub. CONTRIBUTING.md now says an author is a GitHub username, organization or org/team that exists, and the skill staleness reminder no longer calls an author an internal alias. The connector review steering runs the flag without a token, so a reviewer's credentials never reach the trial merge. --- .claude/CLAUDE.md | 3 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/scripts/check_skill_staleness.py | 13 +- .github/scripts/github_logins.py | 156 ++++++++++++++++++ .github/scripts/skill_checks.py | 27 ++- .github/scripts/validate_skills.py | 87 +++++++++- .../workflows/skill-staleness-reminder.yml | 3 +- .github/workflows/validate-skills.yml | 16 +- .kiro/steering/connector-review.md | 4 +- .kiro/steering/project-conventions.md | 3 +- CONTRIBUTING.md | 6 +- tests/test_github_logins.py | 150 +++++++++++++++++ tests/test_validate_skills.py | 134 ++++++++++++++- 13 files changed, 569 insertions(+), 35 deletions(-) create mode 100644 .github/scripts/github_logins.py create mode 100644 tests/test_github_logins.py diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 1e797723e..1410cb48f 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -92,7 +92,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar `.github/workflows/validate-skills.yml` runs `.github/scripts/validate_skills.py` on every pull request, and `.github/workflows/validate-skills-main.yml` runs it again after each push to `main`. `.github/workflows/build-docs.yml` runs `mkdocs build --strict` on pull requests and on `main`. The rules and their reasons are in the "Skill Publishing Rules" section of [CONTRIBUTING.md](../CONTRIBUTING.md). -- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. +- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`) or need the network (`author_login_findings`, the GitHub lookup of new `metadata.author` names); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder and can't reach GitHub. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. - Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn, a placeholder such as `All`, `None` or `*` is an error in any case, and the file can't list one), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. @@ -100,6 +100,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - Like the repository's other checks, a PR runs its own copy of the script, rules and tests, so it could edit them to pass. Code-owner review of `/.github/`, `/tests/` and `/.gitattributes` (`.github/CODEOWNERS`) is the control; don't describe any in-script measure as stopping a determined PR. The post-merge run on `main` catches pull requests that break the rules together, not a pull request that edits the check. - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. - `check_added_content` warns about internal host or tool names only on lines a PR adds, like `scan-aws-identifiers`, so names already on `main` don't warn on every PR. +- `--check-github-logins`, which the PR workflow passes with the job's token, looks up on GitHub each `metadata.author` name a PR adds: `validate_skills.author_login_findings` with the resolver in `.github/scripts/github_logins.py`. The rules are in the `metadata.author` row of CONTRIBUTING.md; `tests/test_github_logins.py` and `tests/test_validate_skills.py` answer for GitHub, so no test reaches it. - With `--base-ref`, `check_history` compares each skill with its merge-base copy: no removed folders, no reuse of a path deleted in `main`'s first-parent history (`git log --first-parent --no-renames --diff-filter=D`, skipped with a warning on a shallow clone), a higher version whenever `content_hash` of the published files changes, one step at a time (next patch, minor or major), and `1.0.0` for a new skill. No folder is ever deleted, not even in an emergency: deprecate it instead. The hash function is pinned by the golden fixture under `content_hash` in `conformance-cases.json`, so any other implementation can prove it computes the same hash. - With or without a base, a deprecated skill can't stay in any `connectors/*/connector.yaml` `skills:` list (`connector_skill_references`). Every `connectors/*/connector.yaml` and `connectors/releases.yaml` is read with `skill_checks.read_yaml`, the reader the frontmatter uses, so a file that isn't UTF-8 or breaks a YAML rule is an error; the cases are in `skill-rules/yaml-conformance-cases.json`, which `tests/test_skill_checks.py` runs. A `connector.yaml` that has a `skills:` value that isn't a list of skill names gets a warning; the names it does list are still checked. One that isn't a mapping is an error from the connector version rules, since it has no version. - `connector_checks.check_history` compares each `connectors//` folder (not `connectors/schema/`) with its merge-base copy, as the catalog publisher compares a connector with its published version, and reads `connector.yaml` as the publisher does (`skill_checks.plain_scalar_value`); without a base only `check_connector` runs. The rules are under "Connector versions" in CONTRIBUTING.md. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 48d052b83..d023811f7 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -64,7 +64,7 @@ before this check existed" in CONTRIBUTING.md. ## Skills — complete if this PR adds or changes a skill - + - [ ] `SKILL.md` frontmatter is valid: `name` matches the directory, `description` present and within length limits, body at most 500 lines. - [ ] The `description` is written from the agent's point of view — scenarios, services, error types, symptoms. diff --git a/.github/scripts/check_skill_staleness.py b/.github/scripts/check_skill_staleness.py index b3df2a1c3..9138de537 100644 --- a/.github/scripts/check_skill_staleness.py +++ b/.github/scripts/check_skill_staleness.py @@ -7,8 +7,9 @@ unlike CHANGELOG dates which are inconsistent across skills. The script reads each ``skills//SKILL.md`` frontmatter for the ``author`` -field (an internal alias, surfaced in the issue as plain text for a maintainer -to route to — it is NOT a GitHub login and is never mentioned or assigned) and +field (GitHub accounts, which validate-skills checks, surfaced in the issue as +plain text for a maintainer to route to — never mentioned or assigned, because an +author can be an organization, a team or someone who isn't a collaborator) and prints: * a human-readable summary to stderr, and @@ -55,10 +56,10 @@ def issue_title(skill: StaleSkill) -> str: def issue_body(skill: StaleSkill, max_age_days: int) -> str: - # The frontmatter `author` is the skill's declared owner, but in this repo it - # is an internal alias, NOT a GitHub login (e.g. "hokang" -> GitHub user - # "howard-m-k"). So it is surfaced as plain text for a maintainer to route to, - # never @-mentioned or used as an assignee. + # The frontmatter `author` is the skill's declared owner: GitHub accounts, + # which validate-skills checks. It is still surfaced as plain text for a + # maintainer to route to, never @-mentioned or used as an assignee, because an + # author can be an organization, a team or someone who isn't a collaborator. listed_author = ", ".join(f"`{a}`" for a in skill.authors) or ( "_none listed in SKILL.md frontmatter_" ) diff --git a/.github/scripts/github_logins.py b/.github/scripts/github_logins.py new file mode 100644 index 000000000..5103785fe --- /dev/null +++ b/.github/scripts/github_logins.py @@ -0,0 +1,156 @@ +"""Look up GitHub accounts with GET /users/, so that metadata.author names one that exists. + +validate_skills.py --check-github-logins uses this module; the rules are in the +metadata.author row of "Skill Publishing Rules" in CONTRIBUTING.md. The token, +from GH_TOKEN or GITHUB_TOKEN, is sent only to api.github.com. +""" + +from __future__ import annotations + +import http.client +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Callable, Protocol + +API_URL = "https://api.github.com" +# Read in this order, as the GitHub CLI reads them. +TOKEN_VARIABLES = ("GH_TOKEN", "GITHUB_TOKEN") +TIMEOUT_SECONDS = 10 +# One retry, for a server error or a request that fails or times out. +ATTEMPTS = 2 +RETRY_DELAY_SECONDS = 1.0 +# Answers that hold for every login in the run: the token was refused, or a rate limit was reached. Any other +# 3xx or 4xx answer, such as a redirect, is about one login only. +RUN_WIDE_STATUSES = frozenset({401, 403, 429}) +# A user or organization record is a few kilobytes. +MAX_RESPONSE_BYTES = 1_048_576 +USER_AGENT = "tools-for-devops-agent-validate-skills" + +FOUND = "found" +MISSING = "missing" +UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class Lookup: + """What GitHub said about one login. + + `status` is FOUND, MISSING (HTTP 404) or UNKNOWN. `account_type` is + GitHub's "User" or "Organization" when the account was found, and `reason` + says why GitHub couldn't answer when the status is UNKNOWN. + """ + + status: str + account_type: str | None = None + reason: str | None = None + + +class Resolver(Protocol): + def lookup(self, login: str) -> Lookup: ... + + +class _RefuseRedirects(urllib.request.HTTPRedirectHandler): + """Turns a redirect into an HTTP error, so the token never goes to another address.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def build_opener(*handlers: urllib.request.BaseHandler) -> urllib.request.OpenerDirector: + """An opener that refuses redirects. The tests pass a handler that answers in place of the network.""" + return urllib.request.build_opener(_RefuseRedirects(), *handlers) + + +class GitHubResolver: + """Looks up logins with GET /users/, once per login per run, ignoring case as GitHub does. + + Once GitHub can't answer a lookup (a refused token, a rate limit, or a + server error or failed request even after the retry), no more requests are + made: each later login is UNKNOWN straight away, with the reason of the + lookup that stopped them. An outage then costs one lookup's timeout and + retry, not one per login, and a rate limit isn't hit again. + """ + + def __init__( + self, + token: str | None, + *, + opener: urllib.request.OpenerDirector | None = None, + sleep: Callable[[float], None] = time.sleep, + ): + self._token = token or None + self._opener = opener or build_opener() + self._sleep = sleep + self._cache: dict[str, Lookup] = {} + # Why GitHub stopped answering, once a lookup has found that it can't. + self._stopped: str | None = None + + @classmethod + def from_environment(cls) -> GitHubResolver: + return cls(next((os.environ[name] for name in TOKEN_VARIABLES if os.environ.get(name)), None)) + + def lookup(self, login: str) -> Lookup: + key = login.lower() + if key not in self._cache: + self._cache[key] = self._fetch(login) + return self._cache[key] + + def _fetch(self, login: str) -> Lookup: + if self._token is None: + return Lookup(UNKNOWN, reason=f"no GitHub token is set in {' or '.join(TOKEN_VARIABLES)}") + if self._stopped is not None: + return Lookup(UNKNOWN, reason=f"an earlier lookup showed that {self._stopped}") + request = urllib.request.Request( + f"{API_URL}/users/{urllib.parse.quote(login, safe='')}", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self._token}", + "User-Agent": USER_AGENT, + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + reason = "GitHub didn't answer" + for attempt in range(ATTEMPTS): + if attempt: + self._sleep(RETRY_DELAY_SECONDS) + try: + with self._opener.open(request, timeout=TIMEOUT_SECONDS) as response: + body = response.read(MAX_RESPONSE_BYTES) + return Lookup(FOUND, account_type=_account_type(body)) + except urllib.error.HTTPError as exc: + exc.close() + if exc.code == 404: + return Lookup(MISSING) + reason = _http_error_reason(exc) + if exc.code < 500: + # A retry wouldn't get another answer within the run. + if exc.code in RUN_WIDE_STATUSES: + self._stopped = reason + return Lookup(UNKNOWN, reason=reason) + except (urllib.error.URLError, http.client.HTTPException, OSError): + # TimeoutError is an OSError, and URLError wraps a failed connection. + reason = "the request to GitHub failed or timed out" + # A server error or a failed request, even after the retry: GitHub isn't answering, so stop asking. + self._stopped = reason + return Lookup(UNKNOWN, reason=reason) + + +def _account_type(body: bytes) -> str | None: + try: + document = json.loads(body) + except ValueError: + return None + kind = document.get("type") if isinstance(document, dict) else None + return kind if isinstance(kind, str) else None + + +def _http_error_reason(exc: urllib.error.HTTPError) -> str: + remaining = exc.headers.get("X-RateLimit-Remaining") if exc.headers else None + if exc.code == 429 or (exc.code == 403 and remaining == "0"): + return "GitHub's API rate limit was reached" + return f"GitHub returned HTTP {exc.code}" diff --git a/.github/scripts/skill_checks.py b/.github/scripts/skill_checks.py index 68a01d294..ef7e0b196 100644 --- a/.github/scripts/skill_checks.py +++ b/.github/scripts/skill_checks.py @@ -315,6 +315,7 @@ class SkillInfo: version_line: int | None = None deprecated_line: int | None = None name_line: int | None = None + author_line: int | None = None def load_rules(rules_dir: Path, gitignore_text: str) -> Rules: @@ -889,6 +890,7 @@ def _check_skill_md(folder: str, data: bytes, rules: Rules, findings: list[Findi version_line=lines.get("version"), deprecated_line=lines.get("deprecated"), name_line=_line(top["name"][0]) if "name" in top else None, + author_line=lines.get("author"), ) @@ -1094,8 +1096,7 @@ def _check_author(metadata: dict, items: dict, lines: dict, metadata_line: int, return if "author" not in metadata: return - parts = _split(metadata["author"]) - if not all(GITHUB_LOGIN_PATTERN.fullmatch(part) or GITHUB_TEAM_PATTERN.fullmatch(part) for part in parts): + if author_names(metadata["author"]) != _split(metadata["author"]): report.error( '"metadata.author" must be one or more GitHub usernames or teams (org/team) separated by commas, ' 'for example "octocat", "octocat, hubot" or "octo-org/octo-team"', @@ -1103,6 +1104,13 @@ def _check_author(metadata: dict, items: dict, lines: dict, metadata_line: int, ) +def author_names(author: str) -> list[str]: + """The names in a metadata.author value that have the form of a GitHub login or org/team, in order.""" + return [ + part for part in _split(author) if GITHUB_LOGIN_PATTERN.fullmatch(part) or GITHUB_TEAM_PATTERN.fullmatch(part) + ] + + def _check_version( metadata: dict, items: dict, lines: dict, metadata_line: int, report: _Report ) -> tuple[int, int, int] | None: @@ -1627,6 +1635,16 @@ def _lenient_version(tree: SkillTree) -> tuple[int, int, int] | None: Fixing a two-part version changes SKILL.md, so the fix itself needs a version above the old one: 2.6 -> 2.6.1, not 2.6.0. """ + text = lenient_metadata_text(tree, "version") + return parse_version(text, allow_two_part=True) if text is not None else None + + +def lenient_metadata_text(tree: SkillTree, key: str) -> str | None: + """A metadata value of a merge-base copy, as PyYAML alone reads it, or None when it isn't text. + + The merge-base copy may predate the stricter frontmatter rules, so they + aren't applied to it. + """ entry = tree.entries.get("SKILL.md") try: match = FRONTMATTER_PATTERN.match(entry.data.decode("utf-8")) if entry else None @@ -1634,10 +1652,9 @@ def _lenient_version(tree: SkillTree) -> tuple[int, int, int] | None: return None scratch = _Report([]) root = _parse_frontmatter(match.group(1) or "", scratch, strict=False) if match else None - metadata = _last_value(root, "metadata") - node = _last_value(metadata, "version") + node = _last_value(_last_value(root, "metadata"), key) if isinstance(node, ScalarNode) and node.tag == STR_TAG: - return parse_version(node.value, allow_two_part=True) + return node.value return None diff --git a/.github/scripts/validate_skills.py b/.github/scripts/validate_skills.py index c2f93c3c7..14ec1d9d5 100644 --- a/.github/scripts/validate_skills.py +++ b/.github/scripts/validate_skills.py @@ -21,10 +21,14 @@ skill-rules/conformance-cases.json through the same rules and stops with exit code 2 if any case gives the wrong result. +With --check-github-logins, the metadata.author names that a pull request adds +are also looked up on GitHub (github_logins.py). + Usage: python3 .github/scripts/validate_skills.py --base-ref origin/main python3 .github/scripts/validate_skills.py # whole tree, no base python3 .github/scripts/validate_skills.py --self-check-only + GH_TOKEN=... python3 .github/scripts/validate_skills.py --base-ref origin/main --check-github-logins Exit codes: 0 every skill passes (warnings allowed), 1 a skill breaks a rule, 2 the check could not run (a Git failure, malformed rule data, or a failed @@ -46,6 +50,7 @@ import yaml import connector_checks as cc +import github_logins import skill_checks as sc SKILLS_DIR = "skills" @@ -373,15 +378,66 @@ def _recommended_skills(root: yaml.Node | None) -> tuple[str | None, list[str]]: return None, names +def author_login_findings( + info: sc.SkillInfo, base: sc.SkillTree | None, resolver: github_logins.Resolver +) -> list[sc.Finding]: + """Findings for the metadata.author names that are new since `base`, each looked up on GitHub. + + A name is new when the skill has no merge-base copy (it is new, or there is + no base) or when the merge-base copy's author line doesn't list it, ignoring + case as GitHub does. Names already on the base branch aren't looked up, so + an account deleted later can't fail a pull request that leaves the author + line alone. A team, org/team, is looked up by its organization. + """ + author = info.metadata.get("author") + if author is None: + return [] + old_author = sc.lenient_metadata_text(base, "author") if base is not None else None + old_names = {name.lower() for name in sc.author_names(old_author)} if old_author is not None else set() + findings: list[sc.Finding] = [] + for name in dict.fromkeys(sc.author_names(author)): + if name.lower() in old_names: + continue + account, is_team, _ = name.partition("/") + lookup = resolver.lookup(account) + severity = "error" + if lookup.status == github_logins.MISSING: + what = f'is a team of "{account}", which isn\'t' if is_team else "isn't" + message = ( + f'"{name}" in metadata.author {what} a GitHub account. Use the GitHub username of whoever maintains ' + "the skill, or their organization or org/team" + ) + elif lookup.status == github_logins.FOUND and is_team and lookup.account_type == "User": + message = ( + f'"{name}" in metadata.author is a team of "{account}", but "{account}" is a GitHub user, not an ' + "organization, so it has no teams" + ) + elif lookup.status == github_logins.UNKNOWN: + severity = "warning" + message = ( + f'"{name}" in metadata.author wasn\'t looked up on GitHub, because {lookup.reason}; ' + "make sure it is a GitHub account" + ) + else: + continue + findings.append(sc.Finding(severity, message, "SKILL.md", info.author_line)) + return findings + + def check_repository( - repo_root: Path, rules: sc.Rules, base: str | None, head_ref: str + repo_root: Path, + rules: sc.Rules, + base: str | None, + head_ref: str, + resolver: github_logins.Resolver | None = None, ) -> tuple[list[SkillResult], set[str] | None]: """Results for every skill at `head_ref`, and the skills touched since `base` (None without a base). With a base, each skill is also compared with its merge-base copy: a folder can't be removed or reuse a retired path, the version must go up when the published files change, and the lines the pull request adds are checked - for internal host and tool names. + for internal host and tool names. With a resolver, the metadata.author + names that are new since the base are looked up on GitHub. """ head_trees, results = read_skills(repo_root, head_ref) results += repository_results(repo_root, head_ref) @@ -408,6 +464,8 @@ def check_repository( findings, info = sc.check_skill(head_trees[name], rules) if base is not None: findings += sc.check_added_content(base_trees.get(name), head_trees[name], rules) + if resolver is not None and info is not None: + findings += author_login_findings(info, base_trees.get(name), resolver) if info is not None and info.metadata.get("deprecated") == "true": for connector in recommended.get(name, []): findings.append( @@ -541,20 +599,34 @@ def _could_not_run(exc: GitError) -> int: return 2 -def main(argv: list[str] | None = None, repo_root: Path | None = None, rules_dir: Path = RULES_DIR) -> int: +def main( + argv: list[str] | None = None, + repo_root: Path | None = None, + rules_dir: Path = RULES_DIR, + login_resolver: github_logins.Resolver | None = None, +) -> int: + """Runs the check. `login_resolver` replaces the GitHub lookup, so tests never reach the network.""" try: - return _run(argv, repo_root, rules_dir) + return _run(argv, repo_root, rules_dir, login_resolver) except Exception as exc: # noqa: BLE001 - any crash means the check didn't run, never a finding. print(f"::error title=Skill publishing rules could not run::{_escape_data(f'{type(exc).__name__}: {exc}')}") print("The check stopped unexpectedly. This is not a finding about the pull request.") return 2 -def _run(argv: list[str] | None, repo_root: Path | None, rules_dir: Path) -> int: +def _run( + argv: list[str] | None, repo_root: Path | None, rules_dir: Path, login_resolver: github_logins.Resolver | None +) -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--base-ref", help="the branch the pull request merges into, such as origin/main") parser.add_argument("--head-ref", default="HEAD", help="the commit to check (default: HEAD)") parser.add_argument("--self-check-only", action="store_true", help="run only the conformance cases") + parser.add_argument( + "--check-github-logins", + action="store_true", + help="look up on GitHub each metadata.author name that a new skill lists or a changed author line adds, " + f"with the token in {' or '.join(github_logins.TOKEN_VARIABLES)}", + ) args = parser.parse_args(argv) repo_root = repo_root or _repo_root() @@ -592,13 +664,16 @@ def _run(argv: list[str] | None, repo_root: Path | None, rules_dir: Path) -> int print("Self-check passed.") return 0 + resolver = None + if args.check_github_logins: + resolver = login_resolver or github_logins.GitHubResolver.from_environment() if excludes is not None: # The cases above test this pull request's rules; its skills are held to the base branch's published set. rules = dataclasses.replace(rules, published_excludes=excludes) try: if args.head_ref == "HEAD" and uncommitted_skill_changes(repo_root): print(f"note: uncommitted changes under {SKILLS_DIR}/ are not checked; commit them first.\n") - results, touched = check_repository(repo_root, rules, base, args.head_ref) + results, touched = check_repository(repo_root, rules, base, args.head_ref, resolver) except GitError as exc: return _could_not_run(exc) return report(results, touched) diff --git a/.github/workflows/skill-staleness-reminder.yml b/.github/workflows/skill-staleness-reminder.yml index f66602582..e6e1153a4 100644 --- a/.github/workflows/skill-staleness-reminder.yml +++ b/.github/workflows/skill-staleness-reminder.yml @@ -91,7 +91,8 @@ jobs: # Build the triage pool: collaborators with admin or maintain permission. # A random one is assigned per issue to spread triage load. They can then # reassign to the actual skill author (see body). The frontmatter author - # is NOT used as an assignee — it is an internal alias, not a GitHub login. + # is not used as an assignee: it can be an organization, a team or someone + # who isn't a collaborator, and none of those can be assigned. gh api "repos/${GITHUB_REPOSITORY}/collaborators?per_page=100" \ --jq '.[] | select(.permissions.admin or .permissions.maintain) | .login' \ > maintainers.txt diff --git a/.github/workflows/validate-skills.yml b/.github/workflows/validate-skills.yml index a467bbd73..268fd27f7 100644 --- a/.github/workflows/validate-skills.yml +++ b/.github/workflows/validate-skills.yml @@ -8,7 +8,8 @@ name: Validate skills # request changes. # # It runs the unit tests for the check first, so a pull request that changes -# the check is tested by the version it brings. +# the check is tested by the version it brings. It also looks up on GitHub each +# metadata.author name the pull request adds (.github/scripts/github_logins.py). # # NOTE: a red check does not block merging on its own. An admin must add the # job's name ("validate-skills") to the required status checks for `main` in @@ -34,7 +35,8 @@ on: # passes when no skill breaks a rule. # Read-only: the job inspects Git objects and reports a status, so pull requests -# from forks work without any elevated permissions. +# from forks work without any elevated permissions. The job's token, read-only +# too, is used only to look up the GitHub accounts that metadata.author names. permissions: contents: read @@ -79,18 +81,22 @@ jobs: - name: Check every skill continue-on-error: ${{ contains(github.event.pull_request.labels.*.name, 'skip-validate-skills') }} # Event values are passed through the environment rather than - # interpolated into the command, and quoted on use. + # interpolated into the command, and quoted on use. --check-github-logins + # looks up each metadata.author name the pull request adds with + # GH_TOKEN; a lookup GitHub can't answer is only a warning. env: BASE_REF: ${{ github.event.pull_request.base.ref }} + GH_TOKEN: ${{ github.token }} run: | # Defensive only: a pull_request event always names its base branch. # Without one, check the whole tree with no base. if [ -z "${BASE_REF}" ]; then - python3 .github/scripts/validate_skills.py --head-ref HEAD + python3 .github/scripts/validate_skills.py --head-ref HEAD --check-github-logins exit $? fi git fetch --no-tags --quiet origin \ "+refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}" python3 .github/scripts/validate_skills.py \ --base-ref "origin/${BASE_REF}" \ - --head-ref HEAD + --head-ref HEAD \ + --check-github-logins diff --git a/.kiro/steering/connector-review.md b/.kiro/steering/connector-review.md index 835a1c53f..bf9f3e161 100644 --- a/.kiro/steering/connector-review.md +++ b/.kiro/steering/connector-review.md @@ -236,7 +236,7 @@ git diff --name-only "origin/main...origin/pr/$N" -- connectors/releases.yaml ( . "$HOME/.cache/connector-review/${N:?set N to the PR number}.env" || exit 1 [ -z "$(git status --porcelain)" ] || { echo "STOP: the trial merge has uncommitted changes" >&2; exit 1; } -python3 .github/scripts/validate_skills.py --base-ref origin/main +env -u GH_TOKEN -u GITHUB_TOKEN python3 .github/scripts/validate_skills.py --base-ref origin/main --check-github-logins python3 .github/scripts/validate_skill_evals.py --base-ref origin/main --head-ref HEAD --pr-number $N --labels "$(gh pr view $N -R $REPO --json labels --jq '[.labels[].name]')" python3 .github/scripts/scan_aws_identifiers.py --base-ref origin/main python3 -m unittest discover -s tests @@ -244,7 +244,7 @@ pip install -q "mkdocs-material==$(git show origin/main:.github/workflows/build- ) ``` -Record the command, exit status, test count, and warning count for each. The unit-test output includes an `::error::The PR checklist check could not run` line; a checklist test prints it on purpose. Judge the tests by the `Ran N tests` and `OK` lines. `scan_aws_identifiers` covers only account and EC2 instance IDs, so sweep the connector and skill files yourself for the strings listed under Ground rules. +Record the command, exit status, test count, and warning count for each. The unit-test output includes an `::error::The PR checklist check could not run` line; a checklist test prints it on purpose. Judge the tests by the `Ran N tests` and `OK` lines. `scan_aws_identifiers` covers only account and EC2 instance IDs, so sweep the connector and skill files yourself for the strings listed under Ground rules. CI passes `--check-github-logins` with its own token, and a `metadata.author` name GitHub doesn't know fails it. Here the flag runs without a token, so the trial merge never gets your credentials: it lists each new name as a warning instead, and you look each one up with `gh api users/`. **3d. Versions.** diff --git a/.kiro/steering/project-conventions.md b/.kiro/steering/project-conventions.md index cc5ade45b..e7793b9e8 100644 --- a/.kiro/steering/project-conventions.md +++ b/.kiro/steering/project-conventions.md @@ -92,7 +92,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar `.github/workflows/validate-skills.yml` runs `.github/scripts/validate_skills.py` on every pull request, and `.github/workflows/validate-skills-main.yml` runs it again after each push to `main`. `.github/workflows/build-docs.yml` runs `mkdocs build --strict` on pull requests and on `main`. The rules and their reasons are in the "Skill Publishing Rules" section of [CONTRIBUTING.md](../../CONTRIBUTING.md). -- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. +- The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`) or need the network (`author_login_findings`, the GitHub lookup of new `metadata.author` names); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder and can't reach GitHub. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. - Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn, a placeholder such as `All`, `None` or `*` is an error in any case, and the file can't list one), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. @@ -100,6 +100,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - Like the repository's other checks, a PR runs its own copy of the script, rules and tests, so it could edit them to pass. Code-owner review of `/.github/`, `/tests/` and `/.gitattributes` (`.github/CODEOWNERS`) is the control; don't describe any in-script measure as stopping a determined PR. The post-merge run on `main` catches pull requests that break the rules together, not a pull request that edits the check. - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. - `check_added_content` warns about internal host or tool names only on lines a PR adds, like `scan-aws-identifiers`, so names already on `main` don't warn on every PR. +- `--check-github-logins`, which the PR workflow passes with the job's token, looks up on GitHub each `metadata.author` name a PR adds: `validate_skills.author_login_findings` with the resolver in `.github/scripts/github_logins.py`. The rules are in the `metadata.author` row of CONTRIBUTING.md; `tests/test_github_logins.py` and `tests/test_validate_skills.py` answer for GitHub, so no test reaches it. - With `--base-ref`, `check_history` compares each skill with its merge-base copy: no removed folders, no reuse of a path deleted in `main`'s first-parent history (`git log --first-parent --no-renames --diff-filter=D`, skipped with a warning on a shallow clone), a higher version whenever `content_hash` of the published files changes, one step at a time (next patch, minor or major), and `1.0.0` for a new skill. No folder is ever deleted, not even in an emergency: deprecate it instead. The hash function is pinned by the golden fixture under `content_hash` in `conformance-cases.json`, so any other implementation can prove it computes the same hash. - With or without a base, a deprecated skill can't stay in any `connectors/*/connector.yaml` `skills:` list (`connector_skill_references`). Every `connectors/*/connector.yaml` and `connectors/releases.yaml` is read with `skill_checks.read_yaml`, the reader the frontmatter uses, so a file that isn't UTF-8 or breaks a YAML rule is an error; the cases are in `skill-rules/yaml-conformance-cases.json`, which `tests/test_skill_checks.py` runs. A `connector.yaml` that has a `skills:` value that isn't a list of skill names gets a warning; the names it does list are still checked. One that isn't a mapping is an error from the connector version rules, since it has no version. - `connector_checks.check_history` compares each `connectors//` folder (not `connectors/schema/`) with its merge-base copy, as the catalog publisher compares a connector with its published version, and reads `connector.yaml` as the publisher does (`skill_checks.plain_scalar_value`); without a base only `check_connector` runs. The rules are under "Connector versions" in CONTRIBUTING.md. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 18feefe97..35fd91702 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,7 +49,7 @@ Follow these guidelines for each tool: #### Skill Metadata -Make sure the frontmatter in the `SKILL.md` file includes a `metadata` block with `version`, `author` (your GitHub user, or your team as `org/team`) and `aws-devops-agent-skills.*` fields (see examples in existing skills) +Make sure the frontmatter in the `SKILL.md` file includes a `metadata` block with `version`, `author` (your GitHub username, your organization, or your team as `org/team`, which must exist on GitHub) and `aws-devops-agent-skills.*` fields (see examples in existing skills) #### Skill Publishing Rules @@ -64,7 +64,7 @@ Skills on `main` are published automatically, so a pull request check ([`.github | `title` | Optional display name. Non-empty, one line, at most 100 characters. | | `license`, `compatibility`, `allowed-tools` | Optional. When present, non-empty text; `compatibility` is at most 500 characters. | | `metadata` | Required. A mapping whose values are all text. Quote any value YAML would otherwise read as a number or true/false, such as `deprecated: "true"`. A key that isn't listed below is a warning, with the nearest known key when there is one. | -| `metadata.author` | Required. One or more GitHub usernames or teams (`org/team`, such as `octo-org/octo-team`) separated by commas, without a leading `@`. | +| `metadata.author` | Required. One or more GitHub usernames, organizations or teams (`org/team`, such as `octo-org/octo-team`) separated by commas, without a leading `@`. Each must be an existing GitHub account: on a pull request, the check looks up each name that a new skill lists or a changed `author` line adds, and a name GitHub doesn't know is an error. A team is looked up by its organization only, because GitHub shows a team only to the organization's members. When GitHub can't answer, because of a rate limit, a server error or a timeout, the name is a warning instead. | | `metadata.version` | Required. `MAJOR.MINOR.PATCH`, such as `"1.2.0"`, with no prefix, suffix or leading zeros, at most 32 characters and at most 10 digits in each part. | | `metadata.summary` | One line, at most 200 characters, shown as the skill's card text in catalogs. A missing or empty summary is a warning for now; it becomes required once every skill has one. | | `metadata.deprecated` | Optional. `"true"` or `"false"`. | @@ -134,6 +134,8 @@ git fetch upstream main python3 .github/scripts/validate_skills.py --base-ref upstream/main ``` +The GitHub account lookup for `metadata.author` runs only with `--check-github-logins`, which the pull request check passes, and it needs a token in `GH_TOKEN` or `GITHUB_TOKEN`, such as `GH_TOKEN="$(gh auth token)"`. Without a token, each name it would look up is a warning. + Before it checks any skill, the script runs the cases in [`conformance-cases.json`](.github/scripts/skill-rules/conformance-cases.json) through the same rules. When you change a rule, add or update a case for it in the same pull request; the YAML rules also have cases in [`yaml-conformance-cases.json`](.github/scripts/skill-rules/yaml-conformance-cases.json), which the unit tests run, and the connector version rules in [`connector-conformance-cases.json`](.github/scripts/skill-rules/connector-conformance-cases.json), which the script runs with the skill cases. `python3 .github/scripts/validate_skills.py --self-check-only` runs only the cases, and `python3 -m unittest discover -s tests -v` runs the check's unit tests. #### Test Your Skill diff --git a/tests/test_github_logins.py b/tests/test_github_logins.py new file mode 100644 index 000000000..14397ce42 --- /dev/null +++ b/tests/test_github_logins.py @@ -0,0 +1,150 @@ +"""Unit tests for .github/scripts/github_logins.py. + +A fake HTTPS handler answers in place of GitHub, so no test reaches the network. +""" + +from __future__ import annotations + +import email.message +import io +import json +import sys +import unittest +import urllib.error +import urllib.request +import urllib.response +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / ".github" / "scripts")) + +import github_logins as gl # noqa: E402 + + +def headers(**values: str) -> email.message.Message: + message = email.message.Message() + for name, value in values.items(): + message[name.replace("_", "-")] = value + return message + + +class FakeGitHub(urllib.request.HTTPSHandler): + """Answers each request with the next response in `answers`: (status, headers, body), or an exception to raise.""" + + def __init__(self, *answers): + super().__init__() + self.answers = list(answers) + self.requests: list[urllib.request.Request] = [] + self.timeouts: list[float] = [] + + def https_open(self, request): + self.requests.append(request) + self.timeouts.append(request.timeout) + answer = self.answers.pop(0) + if isinstance(answer, BaseException): + raise answer + status, answer_headers, body = answer + response = urllib.response.addinfourl(io.BytesIO(body), answer_headers, request.full_url, status) + response.msg = "answer" + return response + + +def resolver(*answers, token: str | None = "secret-token") -> tuple[gl.GitHubResolver, FakeGitHub, list[float]]: + fake = FakeGitHub(*answers) + sleeps: list[float] = [] + return gl.GitHubResolver(token, opener=gl.build_opener(fake), sleep=sleeps.append), fake, sleeps + + +def ok(account_type: str = "User"): + return 200, headers(Content_Type="application/json"), json.dumps({"login": "x", "type": account_type}).encode() + + +class LookupTests(unittest.TestCase): + def test_an_account_is_found_with_its_type(self): + lookups, fake, _ = resolver(ok("Organization")) + self.assertEqual(lookups.lookup("octo-org"), gl.Lookup(gl.FOUND, account_type="Organization")) + request = fake.requests[0] + self.assertEqual(request.full_url, "https://api.github.com/users/octo-org") + self.assertEqual(request.get_header("Authorization"), "Bearer secret-token") + self.assertEqual(request.get_header("Accept"), "application/vnd.github+json") + self.assertTrue(request.get_header("User-agent")) + self.assertEqual(fake.timeouts, [gl.TIMEOUT_SECONDS]) + + def test_each_answer_and_what_it_does_to_later_lookups(self): + missing = (404, headers(), b'{"message": "Not Found"}') + rate_limited = gl.Lookup(gl.UNKNOWN, reason="GitHub's API rate limit was reached") + failed = gl.Lookup(gl.UNKNOWN, reason="the request to GitHub failed or timed out") + + def http(status: int) -> gl.Lookup: + return gl.Lookup(gl.UNKNOWN, reason=f"GitHub returned HTTP {status}") + + moved = (301, headers(Location="https://elsewhere.example/users/octocat"), b"") + # label: (answers to the first lookup, its result, requests it makes, whether later lookups stop) + rows = { + "404": ([missing], gl.Lookup(gl.MISSING), 1, False), + "403 with no requests left": ([(403, headers(X_RateLimit_Remaining="0"), b"{}")], rate_limited, 1, True), + "429": ([(429, headers(Retry_After="60"), b"{}")], rate_limited, 1, True), + "403": ([(403, headers(), b"{}")], http(403), 1, True), + "401": ([(401, headers(), b"{}")], http(401), 1, True), + "422": ([(422, headers(), b"{}")], http(422), 1, False), + "a redirect, not followed": ([moved], http(301), 1, False), + "502, then found": ([(502, headers(), b""), ok()], gl.Lookup(gl.FOUND, account_type="User"), 2, False), + "a 5xx twice": ([(502, headers(), b""), (503, headers(), b"")], http(503), 2, True), + "a timeout twice": ([TimeoutError("timed out")] * 2, failed, 2, True), + "a refused connection twice": ([urllib.error.URLError("connection refused")] * 2, failed, 2, True), + "a reset connection twice": ([ConnectionResetError()] * 2, failed, 2, True), + "a timeout, then not found": ([TimeoutError("timed out"), missing], gl.Lookup(gl.MISSING), 2, False), + } + for label, (answers, expected, requests, stops) in rows.items(): + with self.subTest(label): + lookups, fake, sleeps = resolver(*answers, ok()) + self.assertEqual(lookups.lookup("octocat"), expected) + self.assertEqual(len(fake.requests), requests) + self.assertEqual(sleeps, [gl.RETRY_DELAY_SECONDS] * (requests - 1)) + later = lookups.lookup("hubot") + if stops: + # Nothing more is asked, so an outage costs one lookup's retry, not one per login. + reason = f"an earlier lookup showed that {expected.reason}" + self.assertEqual(later, gl.Lookup(gl.UNKNOWN, reason=reason)) + self.assertEqual(len(fake.requests), requests) + else: + self.assertEqual(later.status, gl.FOUND) + self.assertEqual(len(fake.requests), requests + 1) + urls = {request.full_url for request in fake.requests} + self.assertLessEqual(urls, {f"{gl.API_URL}/users/octocat", f"{gl.API_URL}/users/hubot"}) + + def test_without_a_token_nothing_is_requested(self): + for token in (None, ""): + with self.subTest(token=token): + lookups, fake, _ = resolver(token=token) + self.assertEqual( + lookups.lookup("octocat"), + gl.Lookup(gl.UNKNOWN, reason="no GitHub token is set in GH_TOKEN or GITHUB_TOKEN"), + ) + self.assertEqual(fake.requests, []) + + def test_each_login_is_looked_up_once_ignoring_case(self): + lookups, fake, _ = resolver(ok(), ok()) + self.assertEqual(lookups.lookup("OctoCat"), lookups.lookup("octocat")) + self.assertEqual(len(fake.requests), 1) + + def test_a_body_that_is_not_json_still_counts_as_found(self): + lookups, _, _ = resolver((200, headers(), b"")) + self.assertEqual(lookups.lookup("octocat"), gl.Lookup(gl.FOUND)) + + def test_the_token_comes_from_gh_token_then_github_token(self): + cases = ( + ({"GH_TOKEN": "gh", "GITHUB_TOKEN": "github"}, "gh"), + ({"GH_TOKEN": "", "GITHUB_TOKEN": "github"}, "github"), + ({"GITHUB_TOKEN": "github"}, "github"), + ({}, None), + ) + for environment, token in cases: + with self.subTest(environment=environment): + with mock.patch.dict("os.environ", environment, clear=True): + self.assertEqual(gl.GitHubResolver.from_environment()._token, token) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_validate_skills.py b/tests/test_validate_skills.py index 9d0a7d7f5..b3ec85ddf 100644 --- a/tests/test_validate_skills.py +++ b/tests/test_validate_skills.py @@ -21,6 +21,7 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / ".github" / "scripts")) +import github_logins # noqa: E402 import validate_skills # noqa: E402 GIT_ENV = { @@ -33,10 +34,10 @@ } -def skill_md(name: str, version: str = "1.0.0", extra: str = "") -> str: +def skill_md(name: str, version: str = "1.0.0", extra: str = "", author: str = "octocat") -> str: return ( f"---\nname: {name}\ndescription: Use this skill when testing {name}.\n" - f'metadata:\n author: octocat\n version: "{version}"\n' + f'metadata:\n author: {author}\n version: "{version}"\n' f' summary: "Tests {name}."\n aws-devops-agent-skills.agent-types: "CHAT"\n' f"{extra}---\n\n# {name}\n" ) @@ -74,16 +75,25 @@ def rules_copy(self) -> Path: return target def run_check( - self, *argv: str, env: dict | None = None, rules_dir: Path | None = None, repo_root: Path | None = None + self, + *argv: str, + env: dict | None = None, + rules_dir: Path | None = None, + repo_root: Path | None = None, + login_resolver=None, ) -> tuple[int, str]: # Strict UTF-8, as on a real stdout, so text that can't be printed fails the test. raw = io.BytesIO() out = io.TextIOWrapper(raw, encoding="utf-8", errors="strict", newline="\n") - environ = {k: v for k, v in os.environ.items() if not k.startswith("GITHUB_")} + # As outside GitHub Actions, with no token even if the shell exports one: github_logins reads GH_TOKEN too. + environ = {k: v for k, v in os.environ.items() if not k.startswith("GITHUB_") and k != "GH_TOKEN"} environ.update(env or {}) with mock.patch.dict(os.environ, environ, clear=True), contextlib.redirect_stdout(out): code = validate_skills.main( - list(argv), repo_root=repo_root or self.repo, rules_dir=rules_dir or validate_skills.RULES_DIR + list(argv), + repo_root=repo_root or self.repo, + rules_dir=rules_dir or validate_skills.RULES_DIR, + login_resolver=login_resolver, ) out.flush() return code, raw.getvalue().decode("utf-8") @@ -669,6 +679,120 @@ def test_a_malformed_list_on_the_base_branch_cannot_run(self): self.assertNotIn("skill(s) checked", out) +class StubResolver: + """Answers GitHub lookups from a table: a login it doesn't list is a user that exists.""" + + def __init__(self, answers: dict[str, github_logins.Lookup] | None = None): + self.answers = {login.lower(): lookup for login, lookup in (answers or {}).items()} + self.calls: list[str] = [] + + def lookup(self, login: str) -> github_logins.Lookup: + self.calls.append(login) + return self.answers.get(login.lower(), github_logins.Lookup(github_logins.FOUND, account_type="User")) + + +class RefusingResolver: + def lookup(self, login: str) -> github_logins.Lookup: + raise AssertionError(f"looked up {login} with the flag off") + + +MISSING = github_logins.Lookup(github_logins.MISSING) +ORGANIZATION = github_logins.Lookup(github_logins.FOUND, account_type="Organization") +# The author line of skill_md(). +AUTHOR_LINE = 5 + + +class GitHubLoginTests(RepoTestCase): + def setUp(self): + super().setUp() + self.write("skills/alpha/SKILL.md", skill_md("alpha")) + self.base = self.commit("base") + + def test_a_new_skill_whose_author_is_not_an_account_fails(self): + self.write("skills/beta/SKILL.md", skill_md("beta", author="octocat, no-such-login")) + self.commit("add beta") + resolver = StubResolver({"no-such-login": MISSING}) + code, out = self.run_check("--base-ref", self.base, "--check-github-logins", login_resolver=resolver) + self.assertEqual(code, 1, out) + self.assertIn( + f'FAIL skills/beta/SKILL.md:{AUTHOR_LINE}: "no-such-login" in metadata.author isn\'t a GitHub account', out + ) + self.assertEqual(resolver.calls, ["octocat", "no-such-login"]) + + def test_an_unchanged_author_line_is_not_looked_up(self): + self.write("skills/alpha/SKILL.md", skill_md("alpha", version="1.0.1", extra=' tier: "aws-managed"\n')) + self.commit("touch alpha but not its author") + resolver = StubResolver({"octocat": MISSING}) + code, out = self.run_check("--base-ref", self.base, "--check-github-logins", login_resolver=resolver) + self.assertEqual(code, 0, out) + self.assertEqual(resolver.calls, []) + + def test_a_changed_author_line_looks_up_only_the_names_it_adds(self): + self.write("skills/alpha/SKILL.md", skill_md("alpha", version="1.0.1", author="OctoCat, hubot")) + self.commit("add a co-author") + resolver = StubResolver() + code, out = self.run_check("--base-ref", self.base, "--check-github-logins", login_resolver=resolver) + self.assertEqual(code, 0, out) + self.assertEqual(resolver.calls, ["hubot"]) + + def test_a_team_is_looked_up_by_its_organization(self): + self.write("skills/beta/SKILL.md", skill_md("beta", author="octo-org/octo-team")) + self.write("skills/gamma/SKILL.md", skill_md("gamma", author="gone-org/octo-team")) + self.write("skills/delta/SKILL.md", skill_md("delta", author="octocat/octo-team")) + self.commit("add three team-owned skills") + resolver = StubResolver({"octo-org": ORGANIZATION, "gone-org": MISSING}) + code, out = self.run_check("--base-ref", self.base, "--check-github-logins", login_resolver=resolver) + self.assertEqual(code, 1, out) + self.assertEqual(sorted(resolver.calls), ["gone-org", "octo-org", "octocat"]) + self.assertNotIn("skills/beta", out) + self.assertIn('"gone-org/octo-team" in metadata.author is a team of "gone-org", which isn\'t a GitHub account', out) + self.assertIn('"octocat" is a GitHub user, not an organization, so it has no teams', out) + + def test_a_lookup_github_cannot_answer_is_a_warning(self): + reason = "GitHub's API rate limit was reached" + self.write("skills/beta/SKILL.md", skill_md("beta", author="hubot")) + self.commit("add beta") + resolver = StubResolver({"hubot": github_logins.Lookup(github_logins.UNKNOWN, reason=reason)}) + code, out = self.run_check("--base-ref", self.base, "--check-github-logins", login_resolver=resolver) + self.assertEqual(code, 0, out) + self.assertIn( + f'WARN skills/beta/SKILL.md:{AUTHOR_LINE}: "hubot" in metadata.author wasn\'t looked up on GitHub, ' + f"because {reason}", + out, + ) + + def test_without_a_token_every_new_name_is_a_warning_and_nothing_is_requested(self): + self.write("skills/beta/SKILL.md", skill_md("beta", author="hubot")) + self.commit("add beta") + with mock.patch.object(github_logins.urllib.request.OpenerDirector, "open", side_effect=AssertionError): + code, out = self.run_check("--base-ref", self.base, "--check-github-logins") + self.assertEqual(code, 0, out) + self.assertIn("because no GitHub token is set in GH_TOKEN or GITHUB_TOKEN", out) + + def test_without_the_flag_nothing_is_looked_up(self): + self.write("skills/beta/SKILL.md", skill_md("beta", author="no-such-login")) + self.commit("add beta") + code, out = self.run_check("--base-ref", self.base, login_resolver=RefusingResolver()) + self.assertEqual(code, 0, out) + + def test_without_a_base_every_author_is_looked_up(self): + self.write("skills/beta/SKILL.md", skill_md("beta", author="hubot")) + self.commit("add beta") + resolver = StubResolver() + code, out = self.run_check("--check-github-logins", login_resolver=resolver) + self.assertEqual(code, 0, out) + self.assertEqual(resolver.calls, ["octocat", "hubot"]) + + def test_a_malformed_author_is_reported_once_and_not_looked_up(self): + self.write("skills/beta/SKILL.md", skill_md("beta", author='"octo cat"')) + self.commit("add beta") + resolver = StubResolver() + code, out = self.run_check("--base-ref", self.base, "--check-github-logins", login_resolver=resolver) + self.assertEqual(code, 1, out) + self.assertIn("must be one or more GitHub usernames or teams", out) + self.assertEqual(resolver.calls, []) + + class SelfCheckTests(RepoTestCase): def test_repository_cases_pass(self): code, out = self.run_check("--self-check-only") From 239c903717f12ed67ce2722ed99a7443d99e96b5 Mon Sep 17 00:00:00 2001 From: miscreantmoogly Date: Wed, 11 Nov 2026 00:01:55 -0700 Subject: [PATCH 6/6] feat(ci): make unapproved tag values an error A value missing from vocabulary.json, in aws-services or technical-domains, was only a warning, and warnings show only on the skills a pull request touches. So near-duplicates such as "Database" next to "Databases" reached main, and catalogs, which match values exactly, list them as separate filter options with the skills split between them. Make an unapproved value an error, keeping the alias hint (use "Databases"). A value with no alias now says to use one from vocabulary.json, or to add it there in the same pull request, where a maintainer reviews it. Rename the valid-unapproved-dimension-value-warns-only case to dimension-unapproved-value, now expected to fail. Update the unit tests, the vocabulary.json description, the CONTRIBUTING.md rules row and the project conventions. This must merge only once main has no unapproved values: the tag and vocabulary cleanup, and the Amazon CloudFront and AWS Elemental MediaTailor names from #188. --- .claude/CLAUDE.md | 6 +++--- .../skill-rules/conformance-cases.json | 21 ++++++++++--------- .github/scripts/skill-rules/vocabulary.json | 2 +- .github/scripts/skill_checks.py | 4 ++-- .kiro/steering/project-conventions.md | 6 +++--- CONTRIBUTING.md | 2 +- tests/test_skill_checks.py | 16 ++++++++------ tests/test_validate_skills.py | 10 ++++----- 8 files changed, 36 insertions(+), 31 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 1410cb48f..6506ba0a1 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -95,7 +95,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`) or need the network (`author_login_findings`, the GitHub lookup of new `metadata.author` names); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder and can't reach GitHub. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. -- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn, a placeholder such as `All`, `None` or `*` is an error in any case, and the file can't list one), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. +- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; an unknown value is an error, with the approved spelling when `aliases` has one, a placeholder such as `All`, `None` or `*` is an error in any case, and the file can't list one), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. - The extension allowlist is not duplicated: it is parsed from the `!*.` lines of `skills/.gitignore`, read from the merge base, so a new extension takes effect only once its `.gitignore` change has merged. - Like the repository's other checks, a PR runs its own copy of the script, rules and tests, so it could edit them to pass. Code-owner review of `/.github/`, `/tests/` and `/.gitattributes` (`.github/CODEOWNERS`) is the control; don't describe any in-script measure as stopping a determined PR. The post-merge run on `main` catches pull requests that break the rules together, not a pull request that edits the check. - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. @@ -106,8 +106,8 @@ The `metadata` block with `author` and `version` fields is required, and `summar - `connector_checks.check_history` compares each `connectors//` folder (not `connectors/schema/`) with its merge-base copy, as the catalog publisher compares a connector with its published version, and reads `connector.yaml` as the publisher does (`skill_checks.plain_scalar_value`); without a base only `check_connector` runs. The rules are under "Connector versions" in CONTRIBUTING.md. - Secrets in test fixtures are built at runtime (`"AKIA" + ...`) or stored as base64 in the conformance cases, so no file in the repository contains a literal key or invisible character. Internal names in tests are made up (`corp.example`, `Rivendell`), never real hosts or tools. - `conformance-cases.json` holds language-neutral cases for these rules. Another tool that validates or publishes these skills should give the same `expect` result on the cases for the rules it implements; some rules here are stricter than any other tool's. The script runs every case before checking any skill and exits 2 on a mismatch. A rule change needs a matching case change in the same PR. -- Every rule applies to every skill; nothing is exempted. A rule that existing skills don't meet yet starts as a warning and becomes an error after a cleanup PR brings every skill into line: a missing `metadata.summary`, and dimension values outside the vocabulary. A missing `metadata.aws-devops-agent-skills.agent-types` stays a warning. -- Agent types come from `metadata.aws-devops-agent-skills.agent-types` (comma-separated values from `agent-types.json`, not from `vocabulary.json` like the other two dimensions, and an unknown value is an error rather than a warning). `GENERIC` loads the skill for every agent, so combining it with another value is an error. Without the key a skill installs as `GENERIC`, which warns. +- Every rule applies to every skill; nothing is exempted. A rule that existing skills don't meet yet starts as a warning and becomes an error after a cleanup PR brings every skill into line, as dimension values outside the vocabulary did; a missing `metadata.summary` is still a warning until its cleanup. A missing `metadata.aws-devops-agent-skills.agent-types` stays a warning. +- Agent types come from `metadata.aws-devops-agent-skills.agent-types` (comma-separated values from `agent-types.json`, not from `vocabulary.json` like the other two dimensions; an unknown value is an error in all three). `GENERIC` loads the skill for every agent, so combining it with another value is an error. Without the key a skill installs as `GENERIC`, which warns. - Metadata values are always text, `deprecated` (`"true"`/`"false"`) and `aws-devops-agent-skills.agent-types` included, as the Agent Skills spec and the skill evaluation tool's structure test require. `metadata.tier` is `aws-managed` or `partner-managed`. Unknown top-level and metadata keys warn, naming the nearest known key; a misspelled `aws-devops-agent-skills.` prefix is an error. - Exit codes: 0 pass (warnings allowed), 1 a rule broken, 2 could not run. Never report a 2 as the PR's fault. - Kill switch: a maintainer's `skip-validate-skills` label sets `continue-on-error` on the PR workflow's install, test and check steps, so the check still runs and reports but can't block that PR; the `labeled`/`unlabeled` triggers re-run it. Never skip the job with `if:` instead: a skipped required check counts as passed. For a check that breaks every PR, an admin drops the required context until the fix merges. diff --git a/.github/scripts/skill-rules/conformance-cases.json b/.github/scripts/skill-rules/conformance-cases.json index bf1c746a4..fd4a9d592 100644 --- a/.github/scripts/skill-rules/conformance-cases.json +++ b/.github/scripts/skill-rules/conformance-cases.json @@ -366,6 +366,17 @@ } } }, + { + "id": "dimension-unapproved-value", + "name": "demo", + "expect": "fail", + "error": "\"Amazon Made Up Service\" is not an approved aws-services value", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\n aws-devops-agent-skills.aws-services: \"Amazon Made Up Service\"\n---\n\n# Demo\n" + } + } + }, { "id": "dimension-unknown", "name": "demo", @@ -1879,16 +1890,6 @@ } } }, - { - "id": "valid-unapproved-dimension-value-warns-only", - "name": "demo", - "expect": "pass", - "files": { - "SKILL.md": { - "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\n aws-devops-agent-skills.aws-services: \"CloudWatch\"\n---\n\n# Demo\n" - } - } - }, { "id": "value-other-parsers-read-as-a-number", "name": "demo", diff --git a/.github/scripts/skill-rules/vocabulary.json b/.github/scripts/skill-rules/vocabulary.json index 30c2f68bd..a3e99c1e8 100644 --- a/.github/scripts/skill-rules/vocabulary.json +++ b/.github/scripts/skill-rules/vocabulary.json @@ -1,5 +1,5 @@ { - "description": "Approved values for the metadata.aws-devops-agent-skills.* dimensions, which skill catalogs such as GitHub Pages show as filters. A value not listed here is a warning, not an error. A placeholder such as All, Any, None, N/A, Other, General or *, in any case, is always an error, so neither values nor aliases may list one; the check refuses to run if they do. aliases maps a value in use today to the approved spelling the check suggests instead. agent-types is absent on purpose: it sets which agents load the skill, so its values are the API values in agent-types.json and an unknown one is an error.", + "description": "Approved values for the metadata.aws-devops-agent-skills.* dimensions, which skill catalogs such as GitHub Pages show as filters. A value not listed here is an error; a pull request that needs a new value adds it here, where a maintainer reviews it. A placeholder such as All, Any, None, N/A, Other, General or *, in any case, can never be approved: the check refuses to run if values or aliases list one. aliases maps a value that isn't approved to the approved spelling the check suggests instead. agent-types is absent on purpose: it sets which agents load the skill, so its values are the API values in agent-types.json and an unknown one is an error.", "values": { "aws-services": [ "AWS Backup", diff --git a/.github/scripts/skill_checks.py b/.github/scripts/skill_checks.py index ef7e0b196..9205f0d01 100644 --- a/.github/scripts/skill_checks.py +++ b/.github/scripts/skill_checks.py @@ -1253,9 +1253,9 @@ def _check_dimensions(metadata: dict, items: dict, rules: Rules, report: _Report hint = ( f'use "{rules.aliases[part]}"' if part in rules.aliases - else f"approved values are listed in {RULES_DIR_DISPLAY}/{VOCABULARY_FILE}" + else f"use a value from {RULES_DIR_DISPLAY}/{VOCABULARY_FILE}, or add this one to it for review" ) - report.warning(f'"{part}" is not an approved {dimension} value; {hint}', line=line) + report.error(f'"{part}" is not an approved {dimension} value; {hint}', line=line) # --- Package: the files that are published and installed ----------------------- diff --git a/.kiro/steering/project-conventions.md b/.kiro/steering/project-conventions.md index e7793b9e8..bc25d2e47 100644 --- a/.kiro/steering/project-conventions.md +++ b/.kiro/steering/project-conventions.md @@ -95,7 +95,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - The rules live in `.github/scripts/skill_checks.py` as pure functions over an in-memory skill folder (`SkillTree` of `Entry(path, mode, data)`). `validate_skills.py` reads Git objects, computes the merge base and reports. The only rules it holds look beyond one skill folder (a file or submodule directly under `skills/`, symbolic links and `.gitattributes` lines outside it, a deprecated skill still listed in a `connectors/*/connector.yaml`, and the YAML of the files under `connectors/`) or need the network (`author_login_findings`, the GitHub lookup of new `metadata.author` names); `tests/test_validate_skills.py` covers them, since a conformance case is one skill folder and can't reach GitHub. Keep every per-skill rule in `skill_checks.py`, so the conformance cases and unit tests exercise exactly what CI runs. The connector version rules live the same way in `.github/scripts/connector_checks.py`, pure functions over a connector folder in the same `SkillTree`, with their cases in `skill-rules/connector-conformance-cases.json`, which the script also runs before checking anything, and unit tests in `tests/test_connector_checks.py`. - Every skill is checked on every PR, because skills on `main` are published automatically. Errors from any skill fail; warnings print only for skills the PR touches. - Skills are read from Git objects (`git ls-tree -r -z` plus `git cat-file --batch`), never the working tree, so modes are exact and a local run checks only committed changes. -- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; unknown values only warn, a placeholder such as `All`, `None` or `*` is an error in any case, and the file can't list one), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. +- Rule data lives in `.github/scripts/skill-rules/` and the check fails closed (exit 2) when a file there is malformed: `agent-types.json` (the values `aws-devops-agent-skills.agent-types` may take; an unknown one is an error), `vocabulary.json` (the `aws-services` and `technical-domains` values; an unknown value is an error, with the approved spelling when `aliases` has one, a placeholder such as `All`, `None` or `*` is an error in any case, and the file can't list one), `published-files.json` (root names left out of the published set: `evals/`, `.skilleval.yaml` and `CHANGELOG.md`; `README.md` and `images/` are published; read from the merge base when there is one, like the extension allowlist, so a change to it takes effect only once it has merged), `internal-names.json` (hosts and terms that warn on added lines) and `conformance-cases.json`. - The extension allowlist is not duplicated: it is parsed from the `!*.` lines of `skills/.gitignore`, read from the merge base, so a new extension takes effect only once its `.gitignore` change has merged. - Like the repository's other checks, a PR runs its own copy of the script, rules and tests, so it could edit them to pass. Code-owner review of `/.github/`, `/tests/` and `/.gitattributes` (`.github/CODEOWNERS`) is the control; don't describe any in-script measure as stopping a determined PR. The post-merge run on `main` catches pull requests that break the rules together, not a pull request that edits the check. - Limits: at most 100 published files, each at most 1 MiB, a zip of at most 4,128,768 bytes (4 MiB less a 64 KiB margin), and frontmatter at most 32 KiB. The check warns past 90% of the file count, file size and zip limits. `validate_skills.repository_results` also rejects symbolic links anywhere in the repository and root `.gitattributes` lines that change archived bytes under `skills/`, because skills are published from an archive of the repository, which must hold exactly what Git holds. @@ -106,8 +106,8 @@ The `metadata` block with `author` and `version` fields is required, and `summar - `connector_checks.check_history` compares each `connectors//` folder (not `connectors/schema/`) with its merge-base copy, as the catalog publisher compares a connector with its published version, and reads `connector.yaml` as the publisher does (`skill_checks.plain_scalar_value`); without a base only `check_connector` runs. The rules are under "Connector versions" in CONTRIBUTING.md. - Secrets in test fixtures are built at runtime (`"AKIA" + ...`) or stored as base64 in the conformance cases, so no file in the repository contains a literal key or invisible character. Internal names in tests are made up (`corp.example`, `Rivendell`), never real hosts or tools. - `conformance-cases.json` holds language-neutral cases for these rules. Another tool that validates or publishes these skills should give the same `expect` result on the cases for the rules it implements; some rules here are stricter than any other tool's. The script runs every case before checking any skill and exits 2 on a mismatch. A rule change needs a matching case change in the same PR. -- Every rule applies to every skill; nothing is exempted. A rule that existing skills don't meet yet starts as a warning and becomes an error after a cleanup PR brings every skill into line: a missing `metadata.summary`, and dimension values outside the vocabulary. A missing `metadata.aws-devops-agent-skills.agent-types` stays a warning. -- Agent types come from `metadata.aws-devops-agent-skills.agent-types` (comma-separated values from `agent-types.json`, not from `vocabulary.json` like the other two dimensions, and an unknown value is an error rather than a warning). `GENERIC` loads the skill for every agent, so combining it with another value is an error. Without the key a skill installs as `GENERIC`, which warns. +- Every rule applies to every skill; nothing is exempted. A rule that existing skills don't meet yet starts as a warning and becomes an error after a cleanup PR brings every skill into line, as dimension values outside the vocabulary did; a missing `metadata.summary` is still a warning until its cleanup. A missing `metadata.aws-devops-agent-skills.agent-types` stays a warning. +- Agent types come from `metadata.aws-devops-agent-skills.agent-types` (comma-separated values from `agent-types.json`, not from `vocabulary.json` like the other two dimensions; an unknown value is an error in all three). `GENERIC` loads the skill for every agent, so combining it with another value is an error. Without the key a skill installs as `GENERIC`, which warns. - Metadata values are always text, `deprecated` (`"true"`/`"false"`) and `aws-devops-agent-skills.agent-types` included, as the Agent Skills spec and the skill evaluation tool's structure test require. `metadata.tier` is `aws-managed` or `partner-managed`. Unknown top-level and metadata keys warn, naming the nearest known key; a misspelled `aws-devops-agent-skills.` prefix is an error. - Exit codes: 0 pass (warnings allowed), 1 a rule broken, 2 could not run. Never report a 2 as the PR's fault. - Kill switch: a maintainer's `skip-validate-skills` label sets `continue-on-error` on the PR workflow's install, test and check steps, so the check still runs and reports but can't block that PR; the `labeled`/`unlabeled` triggers re-run it. Never skip the job with `if:` instead: a skipped required check counts as passed. For a check that breaks every PR, an admin drops the required context until the fix merges. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35fd91702..bd72658a6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,7 +70,7 @@ Skills on `main` are published automatically, so a pull request check ([`.github | `metadata.deprecated` | Optional. `"true"` or `"false"`. | | `metadata.tier` | Optional. `"aws-managed"` or `"partner-managed"`: who maintains the skill. | | `metadata.aws-devops-agent-skills.agent-types` | The agents that load the skill, as comma-separated values from [`agent-types.json`](.github/scripts/skill-rules/agent-types.json), such as `"INCIDENT_RCA"`. An unknown value is an error, because this key decides which agents load the skill rather than only how catalogs filter it. `GENERIC` loads the skill for every agent, so it is an error to combine it with another value. Without the key the skill installs as `GENERIC` and loads for every agent, which is a warning. | -| `metadata.aws-devops-agent-skills.*` | Only `agent-types`, `aws-services` and `technical-domains`, each a comma-separated list. Any other key under this prefix, or a misspelling of the prefix, is an error, so a typo can't drop the skill from a filter. For `aws-services` and `technical-domains`, a value not in [`vocabulary.json`](.github/scripts/skill-rules/vocabulary.json) is a warning, with the approved spelling when there is one. A placeholder such as `All`, `Any`, `None`, `N/A`, `Other`, `General` or `*`, in any case, is an error, because catalogs would list it as a filter option of its own: leave the key out instead, and the skill still appears when no filter is chosen. | +| `metadata.aws-devops-agent-skills.*` | Only `agent-types`, `aws-services` and `technical-domains`, each a comma-separated list. Any other key under this prefix, or a misspelling of the prefix, is an error, so a typo can't drop the skill from a filter. For `aws-services` and `technical-domains`, a value not in [`vocabulary.json`](.github/scripts/skill-rules/vocabulary.json) is an error, with the approved spelling when there is one, because catalogs show each value as a filter option and a near-duplicate splits the skills between two. To use a new value, add it to `vocabulary.json` in the same pull request, where a maintainer reviews it. A placeholder such as `All`, `Any`, `None`, `N/A`, `Other`, `General` or `*`, in any case, is an error, because catalogs would list it as a filter option of its own: leave the key out instead, and the skill still appears when no filter is chosen. | The frontmatter must be plain YAML that every parser reads the same way. These are rejected: tabs; a byte order mark anywhere; line breaks other than newlines (U+0085, U+2028, U+2029, or a carriage return without a line feed); `---` anywhere but the closing line; anchors (`&`), aliases (`*`), tags (`!`), directives (`%`) and document markers; flow collections (`[...]` or `{...}`); keys that aren't plain text, `<<` keys, and duplicate keys, at any depth; nesting more than 64 levels deep; unquoted values that some parsers read as another type, such as `1e3`, `yes`, `y` or `2026-11-08`; in a double-quoted value, an escaped UTF-16 surrogate such as `\uD83D`, or an escaped line break before a blank line; a block scalar indentation indicator, such as `|2`; a block scalar whose first line is indented less than a blank line before it; a `#` right after a closing quote or bracket; a quoted value whose next line isn't indented past its key or list item with spaces; and a block scalar as the last key, unless it is `|-` or `>-`. A top-level key other than the Agent Skills specification's fields and `title` is a warning. diff --git a/tests/test_skill_checks.py b/tests/test_skill_checks.py index bebfd271c..61158d75c 100644 --- a/tests/test_skill_checks.py +++ b/tests/test_skill_checks.py @@ -511,14 +511,18 @@ def test_agent_types_report_a_placeholder_as_an_unknown_agent_type(self): fm = frontmatter_with(**{"aws-devops-agent-skills.agent-types": '"All"'}) self.assertOneError(tree(skill_md(fm)), 'unknown value "All"') - def test_unapproved_value_is_a_warning_with_suggestion(self): + def test_unapproved_value_is_an_error_with_suggestion(self): fm = frontmatter_with(**{"aws-devops-agent-skills.aws-services": '"CloudWatch, Amazon S3"'}) skill = tree(skill_md(fm)) - self.assertEqual(self.errors(skill), []) - messages = [w.message for w in self.warnings(skill)] - self.assertEqual(len(messages), 2) - self.assertIn('use "Amazon CloudWatch"', messages[0]) - self.assertIn('"Amazon S3" is not an approved aws-services value', messages[1]) + self.assertEqual(self.warnings(skill), []) + errors = self.errors(skill) + self.assertEqual([e.line for e in errors], [9, 9]) + self.assertEqual(errors[0].message, '"CloudWatch" is not an approved aws-services value; use "Amazon CloudWatch"') + self.assertEqual( + errors[1].message, + '"Amazon S3" is not an approved aws-services value; use a value from ' + ".github/scripts/skill-rules/vocabulary.json, or add this one to it for review", + ) class SummaryAndAgentTypeTests(CheckSkillTestCase): diff --git a/tests/test_validate_skills.py b/tests/test_validate_skills.py index b3ec85ddf..b2487df89 100644 --- a/tests/test_validate_skills.py +++ b/tests/test_validate_skills.py @@ -129,9 +129,9 @@ def test_untouched_skill_errors_still_fail(self): self.assertIn("skills/alpha/SKILL.md:6", out) def test_warnings_are_shown_only_for_touched_skills(self): - off_vocabulary = ' aws-devops-agent-skills.aws-services: "Made Up Service"\n' - self.write("skills/alpha/SKILL.md", skill_md("alpha", extra=off_vocabulary)) - self.write("skills/beta/SKILL.md", skill_md("beta", extra=off_vocabulary)) + unknown_key = ' reviewed-by: "hubot"\n' + self.write("skills/alpha/SKILL.md", skill_md("alpha", extra=unknown_key)) + self.write("skills/beta/SKILL.md", skill_md("beta", extra=unknown_key)) base = self.commit("base") self.write("skills/beta/CHANGELOG.md", "# Changelog\n") self.commit("touch beta without changing its published files") @@ -142,8 +142,8 @@ def test_warnings_are_shown_only_for_touched_skills(self): self.assertIn("1 warning(s) on skills this change doesn't touch are not shown", out) def test_without_a_base_every_warning_is_shown(self): - off_vocabulary = ' aws-devops-agent-skills.aws-services: "Made Up Service"\n' - self.write("skills/alpha/SKILL.md", skill_md("alpha", extra=off_vocabulary)) + unknown_key = ' reviewed-by: "hubot"\n' + self.write("skills/alpha/SKILL.md", skill_md("alpha", extra=unknown_key)) self.commit("base") code, out = self.run_check() self.assertEqual(code, 0, out)