diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index b0a67f97a..fa8ca7d37 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -106,7 +106,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - `conformance-cases.json` holds language-neutral cases for these rules. Another tool that validates or publishes these skills should give the same `expect` result on the cases for the rules it implements; some rules here are stricter than any other tool's. The script runs every case before checking any skill and exits 2 on a mismatch. A rule change needs a matching case change in the same PR. - Every rule applies to every skill; nothing is exempted. A rule that existing skills don't meet yet starts as a warning and becomes an error after a cleanup PR brings every skill into line: a missing `metadata.summary`, and dimension values outside the vocabulary. A missing `metadata.aws-devops-agent-skills.agent-types` stays a warning. - Agent types come from `metadata.aws-devops-agent-skills.agent-types` (comma-separated values from `agent-types.json`, not from `vocabulary.json` like the other two dimensions, and an unknown value is an error rather than a warning). `GENERIC` loads the skill for every agent, so combining it with another value is an error. Without the key a skill installs as `GENERIC`, which warns. -- Metadata values are always text, `deprecated` (`"true"`/`"false"`) and `aws-devops-agent-skills.agent-types` included, as the Agent Skills spec and the skill evaluation tool's structure test require. `metadata.tier` is `aws-managed` or `partner-managed`. Unknown top-level and metadata keys warn, naming the nearest known key; a misspelled `aws-devops-agent-skills.` prefix is an error. +- Metadata values are always text, `deprecated` (`"true"`/`"false"`) and `aws-devops-agent-skills.agent-types` included, as the Agent Skills spec and the skill evaluation tool's structure test require. `metadata.tier` is `aws-managed`, `aws-support-managed` or `partner-managed`. Use `aws-support-managed` only for a skill whose named owner or team has accepted ongoing AWS Support maintenance. The check can't verify that, so before merge a maintainer confirms it through AWS Support's internal ownership process, then adds the `aws-support-managed-verified` label or a comment saying it was verified. Who opened the pull request isn't the test. The PR template's reviewer note has the steps. Unknown top-level and metadata keys warn, naming the nearest known key; a misspelled `aws-devops-agent-skills.` prefix is an error. - Exit codes: 0 pass (warnings allowed), 1 a rule broken, 2 could not run. Never report a 2 as the PR's fault. - Kill switch: a maintainer's `skip-validate-skills` label sets `continue-on-error` on the PR workflow's install, test and check steps, so the check still runs and reports but can't block that PR; the `labeled`/`unlabeled` triggers re-run it. Never skip the job with `if:` instead: a skipped required check counts as passed. For a check that breaks every PR, an admin drops the required context until the fix merges. - PyYAML is the only dependency, pinned by every PyPI hash in `skill-rules/requirements.txt`; actions are pinned by commit SHA. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 800839357..307f7f610 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -66,6 +66,18 @@ before this check existed" in CONTRIBUTING.md. + + - [ ] `SKILL.md` frontmatter is valid: `name` matches the directory, `description` present and within length limits, body at most 500 lines. - [ ] The `description` is written from the agent's point of view — scenarios, services, error types, symptoms. - [ ] Scoped to a specific agent type in `metadata.aws-devops-agent-skills.agent-types` (`INCIDENT_RCA`, `INCIDENT_TRIAGE`, `INCIDENT_MITIGATION`, …), not left `GENERIC` without reason. diff --git a/.github/scripts/skill-rules/conformance-cases.json b/.github/scripts/skill-rules/conformance-cases.json index d8ce5c090..30c2e759a 100644 --- a/.github/scripts/skill-rules/conformance-cases.json +++ b/.github/scripts/skill-rules/conformance-cases.json @@ -1804,6 +1804,27 @@ } } }, + { + "id": "tier-aws-support-managed", + "name": "demo", + "expect": "pass", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\n tier: \"aws-support-managed\"\n---\n\n# Demo\n" + } + } + }, + { + "id": "tier-aws-support-near-miss", + "name": "demo", + "expect": "fail", + "error": "\"metadata.tier\" must be", + "files": { + "SKILL.md": { + "text": "---\nname: demo\ndescription: Use this skill when testing the publishing rules.\nmetadata:\n author: octocat\n version: \"1.0.0\"\n tier: \"aws-support\"\n---\n\n# Demo\n" + } + } + }, { "id": "tier-unknown-value", "name": "demo", diff --git a/.github/scripts/skill_checks.py b/.github/scripts/skill_checks.py index a8b0dc73e..24fac5ede 100644 --- a/.github/scripts/skill_checks.py +++ b/.github/scripts/skill_checks.py @@ -42,7 +42,7 @@ MAX_COMPATIBILITY_LENGTH = 500 # The skill evaluation tool's structure test warns above this many body lines. MAX_BODY_LINES = 500 -TIERS = ("aws-managed", "partner-managed") +TIERS = ("aws-managed", "aws-support-managed", "partner-managed") VERSION_PATTERN = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)") # At most 10 digits in each part, so a version is at most 32 characters and @@ -829,8 +829,9 @@ def _check_deprecated(metadata: dict, lines: dict, report: _Report) -> None: def _check_tier(metadata: dict, lines: dict, report: _Report) -> None: tier = metadata.get("tier") if tier is not None and tier not in TIERS: + allowed = ", ".join(f'"{t}"' for t in TIERS[:-1]) + f' or "{TIERS[-1]}"' report.error( - '"metadata.tier" must be "aws-managed" or "partner-managed": who maintains the skill', + f'"metadata.tier" must be {allowed}: who maintains the skill', line=lines["tier"], ) diff --git a/.kiro/steering/project-conventions.md b/.kiro/steering/project-conventions.md index 5f792c97c..e81048263 100644 --- a/.kiro/steering/project-conventions.md +++ b/.kiro/steering/project-conventions.md @@ -106,7 +106,7 @@ The `metadata` block with `author` and `version` fields is required, and `summar - `conformance-cases.json` holds language-neutral cases for these rules. Another tool that validates or publishes these skills should give the same `expect` result on the cases for the rules it implements; some rules here are stricter than any other tool's. The script runs every case before checking any skill and exits 2 on a mismatch. A rule change needs a matching case change in the same PR. - Every rule applies to every skill; nothing is exempted. A rule that existing skills don't meet yet starts as a warning and becomes an error after a cleanup PR brings every skill into line: a missing `metadata.summary`, and dimension values outside the vocabulary. A missing `metadata.aws-devops-agent-skills.agent-types` stays a warning. - Agent types come from `metadata.aws-devops-agent-skills.agent-types` (comma-separated values from `agent-types.json`, not from `vocabulary.json` like the other two dimensions, and an unknown value is an error rather than a warning). `GENERIC` loads the skill for every agent, so combining it with another value is an error. Without the key a skill installs as `GENERIC`, which warns. -- Metadata values are always text, `deprecated` (`"true"`/`"false"`) and `aws-devops-agent-skills.agent-types` included, as the Agent Skills spec and the skill evaluation tool's structure test require. `metadata.tier` is `aws-managed` or `partner-managed`. Unknown top-level and metadata keys warn, naming the nearest known key; a misspelled `aws-devops-agent-skills.` prefix is an error. +- Metadata values are always text, `deprecated` (`"true"`/`"false"`) and `aws-devops-agent-skills.agent-types` included, as the Agent Skills spec and the skill evaluation tool's structure test require. `metadata.tier` is `aws-managed`, `aws-support-managed` or `partner-managed`. Use `aws-support-managed` only for a skill whose named owner or team has accepted ongoing AWS Support maintenance. The check can't verify that, so before merge a maintainer confirms it through AWS Support's internal ownership process, then adds the `aws-support-managed-verified` label or a comment saying it was verified. Who opened the pull request isn't the test. The PR template's reviewer note has the steps. Unknown top-level and metadata keys warn, naming the nearest known key; a misspelled `aws-devops-agent-skills.` prefix is an error. - Exit codes: 0 pass (warnings allowed), 1 a rule broken, 2 could not run. Never report a 2 as the PR's fault. - Kill switch: a maintainer's `skip-validate-skills` label sets `continue-on-error` on the PR workflow's install, test and check steps, so the check still runs and reports but can't block that PR; the `labeled`/`unlabeled` triggers re-run it. Never skip the job with `if:` instead: a skipped required check counts as passed. For a check that breaks every PR, an admin drops the required context until the fix merges. - PyYAML is the only dependency, pinned by every PyPI hash in `skill-rules/requirements.txt`; actions are pinned by commit SHA. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2dfa75f53..85b56616d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ Skills on `main` are published automatically, so a pull request check ([`.github | `metadata.version` | Required. `MAJOR.MINOR.PATCH`, such as `"1.2.0"`, with no prefix, suffix or leading zeros, at most 32 characters and at most 10 digits in each part. | | `metadata.summary` | One line, at most 200 characters, shown as the skill's card text in catalogs. A missing or empty summary is a warning for now; it becomes required once every skill has one. | | `metadata.deprecated` | Optional. `"true"` or `"false"`. | -| `metadata.tier` | Optional. `"aws-managed"` or `"partner-managed"`: who maintains the skill. | +| `metadata.tier` | Optional. `"aws-managed"`, `"aws-support-managed"` or `"partner-managed"`: who maintains the skill. Use `aws-support-managed` only for a skill whose named owner or team has accepted ongoing AWS Support maintenance. The check can't verify that, so before merge a maintainer confirms it through AWS Support's internal ownership process, then adds the `aws-support-managed-verified` label or a comment saying it was verified. Who opened the pull request isn't the test. Skills only: connector tiers are separate. | | `metadata.aws-devops-agent-skills.agent-types` | The agents that load the skill, as comma-separated values from [`agent-types.json`](.github/scripts/skill-rules/agent-types.json), such as `"INCIDENT_RCA"`. An unknown value is an error, because this key decides which agents load the skill rather than only how catalogs filter it. `GENERIC` loads the skill for every agent, so it is an error to combine it with another value. Without the key the skill installs as `GENERIC` and loads for every agent, which is a warning. | | `metadata.aws-devops-agent-skills.*` | Only `agent-types`, `aws-services` and `technical-domains`, each a comma-separated list. Any other key under this prefix, or a misspelling of the prefix, is an error, so a typo can't drop the skill from a filter. For `aws-services` and `technical-domains`, a value not in [`vocabulary.json`](.github/scripts/skill-rules/vocabulary.json) is a warning, with the approved spelling when there is one. | diff --git a/tests/test_skill_checks.py b/tests/test_skill_checks.py index 897108728..509439d20 100644 --- a/tests/test_skill_checks.py +++ b/tests/test_skill_checks.py @@ -453,11 +453,16 @@ def test_version_length_limits(self): self.assertEqual(self.errors(tree(skill_md(frontmatter_with(version=long_version)))), []) def test_tier_values(self): - for good in ('"aws-managed"', '"partner-managed"'): + for good in ('"aws-managed"', '"aws-support-managed"', '"partner-managed"'): with self.subTest(good=good): self.assertEqual(self.errors(tree(skill_md(frontmatter_with(tier=good)))), []) - self.assertOneError(tree(skill_md(frontmatter_with(tier='"community"'))), '"metadata.tier" must be') - self.assertOneError(tree(skill_md(frontmatter_with(tier='"AWS-managed"'))), '"metadata.tier" must be') + self.assertOneError( + tree(skill_md(frontmatter_with(tier='"community"'))), + '"metadata.tier" must be "aws-managed", "aws-support-managed" or "partner-managed"', + ) + for bad in ('"AWS-managed"', '"aws-support"', '"AWS-Support-managed"'): + with self.subTest(bad=bad): + self.assertOneError(tree(skill_md(frontmatter_with(tier=bad))), '"metadata.tier" must be') def test_spec_text_fields_must_not_be_empty(self): for key in ("license", "compatibility", "allowed-tools"): diff --git a/tests/test_validate_skill_evals.py b/tests/test_validate_skill_evals.py index 9602575e0..475b46d0d 100644 --- a/tests/test_validate_skill_evals.py +++ b/tests/test_validate_skill_evals.py @@ -176,7 +176,7 @@ def test_functional_alone_waives_it_with_the_other_results_committed(self): def test_the_tier_plays_no_part(self): """The waiver follows the claim, not who maintains the skill.""" - for tier in ("partner-managed", "aws-managed", None): + for tier in ("partner-managed", "aws-managed", "aws-support-managed", None): with self.subTest(tier=tier, claims="all three"): name = f"claimed-{tier}" self.add_skill(name, tier=tier)