diff --git a/.github/workflows/skillspector.yml b/.github/workflows/skillspector.yml index 35d5dd6..98e937e 100644 --- a/.github/workflows/skillspector.yml +++ b/.github/workflows/skillspector.yml @@ -4,6 +4,8 @@ name: SkillSpector # Static NVIDIA SkillSpector scan of every AI primitive. "Scan AI primitives" # reports on every PR so it can be a required check; it only builds and scans # when the PR touches AI primitives or the scanner (scripts/skillspector-changes.sh). +# Findings surface as checks only: this job's gate, the Code Scanning "SkillSpector" +# check from the SARIF upload, and the run summary. Nothing is posted as a PR comment. on: pull_request: push: @@ -31,8 +33,6 @@ jobs: permissions: contents: read security-events: write - outputs: - scanned: ${{ steps.changes.outputs.relevant }} steps: - name: Harden runner uses: DevSecNinja/.github/actions/harden-runner@27fd66740c629ea38380dcd3bf93e7bc74c54247 # v1.9.0 @@ -97,53 +97,3 @@ jobs: path: ${{ runner.temp }}/skillspector-reports if-no-files-found: warn retention-days: 14 - - comment: - name: Post SkillSpector report - needs: scan - # Always reports on PRs: a skipped job blocks tools that require every check to succeed. - if: ${{ !cancelled() && github.event_name == 'pull_request' }} - runs-on: ubuntu-24.04 - timeout-minutes: 5 - permissions: - contents: read - pull-requests: write - env: - # Only same-repository PRs that were scanned get a comment; fork PRs have a read-only token. - POST: ${{ needs.scan.outputs.scanned == 'true' && !github.event.pull_request.head.repo.fork }} - # A PR that stopped changing AI primitives marks its earlier comment as stale instead. - CLEAR: ${{ needs.scan.outputs.scanned == 'false' && !github.event.pull_request.head.repo.fork }} - steps: - - name: Harden runner - uses: DevSecNinja/.github/actions/harden-runner@27fd66740c629ea38380dcd3bf93e7bc74c54247 # v1.9.0 - - - name: Checkout toolkit - if: env.POST == 'true' || env.CLEAR == 'true' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Download reports - if: env.POST == 'true' - # A missing artifact still produces a "no readable results" comment below. - continue-on-error: true - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: skillspector-reports - path: ${{ runner.temp }}/skillspector-reports - - - name: Create or update the PR comment - if: env.POST == 'true' - env: - GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: >- - python3 scripts/skillspector-report.py "$RUNNER_TEMP/skillspector-reports/repository/summary.json" - --comment-pr "$PR_NUMBER" - - - name: Mark an earlier PR comment as not needed - if: env.CLEAR == 'true' - env: - GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: python3 scripts/skillspector-report.py --not-needed-pr "$PR_NUMBER" diff --git a/.lefthook.toml b/.lefthook.toml index b691c54..40b38b3 100644 --- a/.lefthook.toml +++ b/.lefthook.toml @@ -39,7 +39,7 @@ run = "PYTHONDONTWRITEBYTECODE=1 sh scripts/py.sh -m unittest discover -s tests [pre-commit.commands.hook-git-env] run = "PYTHONDONTWRITEBYTECODE=1 sh scripts/py.sh -m unittest discover -s tests -p 'test_hook_git_env.py'" -# SkillSpector gate and report: severity policy, reviewed acceptances, PR comment. +# SkillSpector gate and report: severity policy, reviewed acceptances, run summary and SARIF. [pre-commit.commands.skillspector-gate] run = "PYTHONDONTWRITEBYTECODE=1 sh scripts/py.sh -m unittest discover -s tests -p 'test_skillspector*.py'" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 01b69fc..6ac2f20 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -206,19 +206,17 @@ Scans run in a non-root, read-only container with no network, no credentials, no LLM and bounded resources, with a three-minute timeout per target. A passing static scan is not a safety guarantee: OSV lookups use the offline fallback. -**Where to read the report.** - -- A **PR comment** (updated in place on every re-run) shows counts by severity, - the top findings with file/line links, accepted items and links to the run. - Comments are posted only on PRs from this repository; fork PRs get a read-only - token. If a later push stops changing AI primitives, the comment is marked - "Not needed" so stale results don't linger. -- The **Actions run summary** lists up to 100 findings and every analysis - limitation. -- **Security > Code scanning** (tool: SkillSpector) holds every finding as - SARIF. GitHub ignores SARIF suppressions, so accepted findings are uploaded - as `/accepted` notes without a security severity. They stay visible but - can't trip Code Scanning's own severity threshold. +**Where to read the report.** Findings show up as checks, not as a PR comment. + +- The **Scan AI primitives** check is the severity gate described above. +- The **SkillSpector** check (Code Scanning, from the SARIF upload) annotates + findings on the PR's changed lines. **Security > Code scanning** (tool: + SkillSpector) holds every finding. GitHub ignores SARIF suppressions, so + accepted findings are uploaded as `/accepted` notes without a security + severity. They stay visible but can't trip Code Scanning's own severity + threshold, so an accepted finding fails neither check. +- The **Actions run summary** shows counts by severity, up to 100 findings with + file/line links, and every analysis limitation. - The `skillspector-reports` artifact keeps raw JSON, scanner logs, `summary.json` and `findings.sarif` for 14 days. Reports can quote source text. @@ -246,9 +244,7 @@ just to get a green check. **Making it a required check.** In **Settings > Rules > Rulesets**, add a status check rule for the default branch requiring **Scan AI primitives** (source: GitHub Actions). Because the job always reports, PRs that don't touch AI -primitives aren't left waiting. Don't require **Post SkillSpector report**: it -only posts on same-repository PRs that were scanned, and passes as a no-op -otherwise. First, stop the +primitives aren't left waiting. First, stop the [Generate Index workflow](.github/workflows/generate-index.yml) from pushing `[skip ci]` commits to PR branches with `GITHUB_TOKEN`. Those commits don't trigger CI, so a required check would stay pending on the new head. diff --git a/scripts/skillspector-report.py b/scripts/skillspector-report.py index 720287a..a3110d8 100755 --- a/scripts/skillspector-report.py +++ b/scripts/skillspector-report.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Render SkillSpector gate results: Actions summary, PR comment and SARIF. +"""Render SkillSpector gate results: Actions run summary and SARIF for Code Scanning. Reads summary.json from scripts/scan-skills.py. Never changes the gate result. """ @@ -11,18 +11,15 @@ import os from pathlib import Path import re -import subprocess import sys from urllib.parse import quote, urlsplit SEVERITIES = ("CRITICAL", "HIGH", "MEDIUM", "LOW") BLOCKING = ("CRITICAL", "HIGH") -MARKER = "" SECURITY_SCORES = {"LOW": "3.0", "MEDIUM": "5.0", "HIGH": "8.0", "CRITICAL": "9.5"} LEVELS = {"LOW": "note", "MEDIUM": "warning", "HIGH": "error", "CRITICAL": "error"} VERDICTS = {0: "✅ Passed", 1: "❌ Blocked: HIGH/CRITICAL findings", 2: "⚠️ Failed: scan errors"} -NOT_NEEDED = "➖ Not needed" ESCAPES = {ord(character): f"&#{ord(character)};" for character in "\\`*_{}[]()!|<>&\"'"} @@ -95,12 +92,11 @@ def finding_table(items: list[dict], limit: int) -> list[str]: return lines -def render(summary: dict, limit: int, pr: int | None = None) -> str: +def render(summary: dict, limit: int) -> str: code = summary["exit_code"] version = summary.get("skillspector_version") or "unknown" revision = (os.environ.get("GITHUB_SHA") or "")[:7] lines = [ - MARKER, f"## SkillSpector report: {VERDICTS.get(code, VERDICTS[2])}", "", f"SkillSpector {cell(version, 32)} static scan (no LLM, no network) of " @@ -112,7 +108,7 @@ def render(summary: dict, limit: int, pr: int | None = None) -> str: ] items = findings(summary) if items: - lines.extend(["", "### Top findings" if limit <= 10 else "### Findings", "", *finding_table(items, limit)]) + lines.extend(["", "### Findings", "", *finding_table(items, limit)]) if summary["errors"]: lines.extend(["", "### Errors", ""]) lines.extend(f"- {cell(item['target'], 120)}: {cell(item['error'], 600)}" for item in summary["errors"][:limit]) @@ -139,7 +135,7 @@ def render(summary: dict, limit: int, pr: int | None = None) -> str: links = [] alerts = github_url("security", "code-scanning") if alerts: - query = quote(f"is:open tool:SkillSpector{f' pr:{pr}' if pr else ''}") + query = quote("is:open tool:SkillSpector") links.append(f"[Code Scanning alerts]({alerts}?query={query})") run = os.environ.get("GITHUB_RUN_ID") if run and github_url(): @@ -226,66 +222,14 @@ def analysis_complete(summary: dict) -> bool: ) -def gh(*args: str, payload: dict | None = None) -> str: - result = subprocess.run( - ["gh", "api", *args], input=json.dumps(payload) if payload else None, - capture_output=True, text=True, check=False, timeout=60, - ) - if result.returncode: - raise RuntimeError(f"gh api {args[0]} failed: {result.stderr.strip()[:500]}") - return result.stdout - - -def upsert_comment(pr: int, body: str, create: bool = True) -> None: - """Create the report comment or update the bot's previous one in place.""" - repository = os.environ["GITHUB_REPOSITORY"] - existing = gh( - "--paginate", f"repos/{repository}/issues/{pr}/comments", "--jq", - f'.[] | select(.user.login == "github-actions[bot]" and (.body | startswith("{MARKER}"))) | .id', - ).split() - if existing: - gh("--method", "PATCH", f"repos/{repository}/issues/comments/{existing[0]}", "--input", "-", - payload={"body": body}) - elif create: - gh("--method", "POST", f"repos/{repository}/issues/{pr}/comments", "--input", "-", - payload={"body": body}) - - -def not_needed() -> str: - revision = (os.environ.get("GITHUB_SHA") or "")[:7] - return (f"{MARKER}\n## SkillSpector report: {NOT_NEEDED}\n\n" - "This PR no longer changes AI primitives or the scanner" - + (f" at `{revision}`" if revision else "") + ", so it wasn't scanned. " - "Earlier results in this comment no longer apply.\n") - - def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("summary", type=Path, nargs="?", help="summary.json written by scan-skills.py") + parser.add_argument("summary", type=Path, help="summary.json written by scan-skills.py") parser.add_argument("--markdown", type=Path, help="Append the detailed Markdown summary here.") parser.add_argument("--sarif", type=Path, help="Write SARIF 2.1.0 for GitHub Code Scanning here.") - parser.add_argument("--comment-pr", type=int, help="Create or update the PR summary comment.") - parser.add_argument("--not-needed-pr", type=int, - help="Mark an existing PR comment as not needed after the PR stops changing AI primitives.") args = parser.parse_args() - if args.not_needed_pr: - if args.summary or args.markdown or args.sarif or args.comment_pr: - parser.error("--not-needed-pr takes no summary or other outputs.") - try: - upsert_comment(args.not_needed_pr, not_needed(), create=False) - except (KeyError, RuntimeError, subprocess.TimeoutExpired) as error: - print(f"SkillSpector report failed: {error}", file=sys.stderr) - return 2 - return 0 - if not args.summary: - parser.error("a summary path is required.") try: if not args.summary.is_file(): - if args.comment_pr: - # The scan job has already failed; say so on the PR instead of going silent. - run = github_url("actions", "runs", quote(os.environ.get("GITHUB_RUN_ID", ""), safe="")) - upsert_comment(args.comment_pr, f"{MARKER}\n## SkillSpector report: {VERDICTS[2]}\n\n" - f"The scan produced no readable results; see the [workflow run]({run}).\n") raise ValueError(f"No summary at {args.summary}.") summary = json.loads(args.summary.read_text(encoding="utf-8")) if args.markdown: @@ -293,9 +237,7 @@ def main() -> int: stream.write(render(summary, limit=100)) if args.sarif: args.sarif.write_text(json.dumps(build_sarif(summary), indent=2) + "\n", encoding="utf-8") - if args.comment_pr: - upsert_comment(args.comment_pr, render(summary, limit=10, pr=args.comment_pr)) - except (OSError, KeyError, ValueError, RuntimeError, subprocess.TimeoutExpired) as error: + except (OSError, KeyError, ValueError) as error: print(f"SkillSpector report failed: {error}", file=sys.stderr) return 2 return 0 diff --git a/tests/test_skillspector_report.py b/tests/test_skillspector_report.py index faa59a4..88afe61 100644 --- a/tests/test_skillspector_report.py +++ b/tests/test_skillspector_report.py @@ -1,4 +1,4 @@ -"""Tests for SkillSpector report rendering, SARIF export and PR comment upserts.""" +"""Tests for SkillSpector run-summary rendering and SARIF export.""" import importlib.util import json @@ -47,14 +47,13 @@ def setUp(self): patcher.start() self.addCleanup(patcher.stop) - def test_comment_counts_orders_blocking_first_and_links_sources(self): + def test_summary_counts_orders_blocking_first_and_links_sources(self): data = summary( finding("MEDIUM", "M1"), finding("HIGH", "H1", accepted="Reviewed deny-list."), finding("CRITICAL", "C1", line=9), finding("LOW", "L1"), exit_code=1, ) - body = report.render(data, limit=10, pr=7) - self.assertTrue(body.startswith(report.MARKER)) - self.assertIn("❌ Blocked", body) + body = report.render(data, limit=10) + self.assertTrue(body.startswith("## SkillSpector report: ❌ Blocked")) self.assertIn("| CRITICAL | 1 | 0 | 1 |", body) self.assertIn("| HIGH | 1 | 1 | 0 |", body) self.assertIn("| MEDIUM | 1 | - | - |", body) @@ -63,7 +62,7 @@ def test_comment_counts_orders_blocking_first_and_links_sources(self): self.assertIn("Accepted: Reviewed deny-list.", rows[3]) self.assertTrue([line for line in body.splitlines() if line.startswith("| ")][-1].startswith("| LOW | L1")) self.assertIn("https://github.com/owner/repo/blob/abc1234def/.apm/skills/x/SKILL.md#L9", body) - self.assertIn("security/code-scanning?query=is%3Aopen%20tool%3ASkillSpector%20pr%3A7", body) + self.assertIn("security/code-scanning?query=is%3Aopen%20tool%3ASkillSpector)", body) self.assertIn("https://github.com/owner/repo/actions/runs/42", body) def test_untrusted_text_cannot_inject_markup_mentions_or_links(self): @@ -156,43 +155,11 @@ def test_errors_mark_sarif_execution_unsuccessful(self): self.assertIn("t: boom", invocation["toolExecutionNotifications"][0]["message"]["text"]) -class CommentTests(unittest.TestCase): - def setUp(self): - patcher = patch.dict(os.environ, ENV) - patcher.start() - self.addCleanup(patcher.stop) - - def test_existing_bot_comment_is_updated_in_place(self): - with patch.object(report, "gh", side_effect=["101\n", ""]) as gh: - report.upsert_comment(7, "body") - self.assertIn("repos/owner/repo/issues/7/comments", gh.call_args_list[0].args) - self.assertEqual(gh.call_args_list[1].args[:3], ("--method", "PATCH", "repos/owner/repo/issues/comments/101")) - self.assertEqual(gh.call_args_list[1].kwargs["payload"], {"body": "body"}) - - def test_first_run_creates_a_comment(self): - with patch.object(report, "gh", side_effect=["", ""]) as gh: - report.upsert_comment(7, "body") - self.assertEqual(gh.call_args_list[1].args[:3], ("--method", "POST", "repos/owner/repo/issues/7/comments")) - - def test_not_needed_updates_only_an_existing_comment(self): - with patch.object(report, "gh", side_effect=["101\n", ""]) as gh, \ - patch("sys.argv", ["report", "--not-needed-pr", "7"]): - self.assertEqual(report.main(), 0) - self.assertEqual(gh.call_args_list[1].args[:2], ("--method", "PATCH")) - body = gh.call_args_list[1].kwargs["payload"]["body"] - self.assertTrue(body.startswith(report.MARKER)) - self.assertIn("➖ Not needed", body) - with patch.object(report, "gh", side_effect=[""]) as gh, \ - patch("sys.argv", ["report", "--not-needed-pr", "7"]): - self.assertEqual(report.main(), 0) - self.assertEqual(gh.call_count, 1) - - def test_missing_summary_still_comments_and_fails(self): - with tempfile.TemporaryDirectory() as temporary, \ - patch.object(report, "upsert_comment") as upsert, patch("sys.stderr"), \ - patch("sys.argv", ["report", str(Path(temporary) / "summary.json"), "--comment-pr", "7"]): +class CliTests(unittest.TestCase): + def test_missing_summary_fails(self): + with tempfile.TemporaryDirectory() as temporary, patch("sys.stderr"), \ + patch("sys.argv", ["report", str(Path(temporary) / "summary.json")]): self.assertEqual(report.main(), 2) - self.assertIn("no readable results", upsert.call_args.args[1]) def test_cli_writes_markdown_and_sarif(self): with tempfile.TemporaryDirectory() as temporary: