diff --git a/.github/workflows/markdown-validation.yml b/.github/workflows/markdown-validation.yml index bb40858..31d8cfb 100644 --- a/.github/workflows/markdown-validation.yml +++ b/.github/workflows/markdown-validation.yml @@ -12,6 +12,7 @@ on: - 'tests/**' - 'scripts/generate-index.sh' - 'scripts/apply-apm-overrides.py' + - 'scripts/scan-skills.py' - 'apm.yml' - 'apm-overrides.json' - '.github/agents/**' @@ -27,6 +28,7 @@ on: - 'tests/**' - 'scripts/generate-index.sh' - 'scripts/apply-apm-overrides.py' + - 'scripts/scan-skills.py' - 'apm.yml' - 'apm-overrides.json' - '.github/agents/**' @@ -79,6 +81,9 @@ jobs: - name: Check vendored primitives keep their local overrides run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_overrides.py' -v + - name: Test SkillSpector severity gate + run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_skillspector.py' -v + consumer-install: name: Check Consumer Install runs-on: ubuntu-latest diff --git a/.lefthook.toml b/.lefthook.toml index faaf18b..9350678 100644 --- a/.lefthook.toml +++ b/.lefthook.toml @@ -25,6 +25,10 @@ run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_ [pre-commit.commands.apm-overrides] run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_overrides.py'" +# SkillSpector gate: severity policy, report contract and reviewed acceptances. +[pre-commit.commands.skillspector-gate] +run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_skillspector.py'" + # Regenerate INDEX.md / README index from .apm/ and re-stage the result, so the # generated index never drifts from the primitives (the common contributor miss). [pre-commit.commands.generate-index] diff --git a/scripts/scan-skills.py b/scripts/scan-skills.py new file mode 100755 index 0000000..fb12fc2 --- /dev/null +++ b/scripts/scan-skills.py @@ -0,0 +1,520 @@ +#!/usr/bin/env python3 +"""Run offline SkillSpector containers and gate individual finding severities. + +Writes summary.json for rendering; exits 0 (pass), 1 (blocking finding) or 2 (error). +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path, PurePosixPath +import re +import stat +import subprocess +import sys + + +SEVERITIES = ("LOW", "MEDIUM", "HIGH", "CRITICAL") +BLOCKING = ("HIGH", "CRITICAL") +SCAN_TIMEOUT = "180s" +CLIENT_TIMEOUT = 300 +ACCEPTED_FILE = ".github/skillspector-accepted.json" +# Directories whose children are scanned one by one, then whole extra roots. +CONTENT_ROOTS = (".apm", ".agents") +EXTRA_ROOTS = ( + ".github/agents", + ".github/prompts", + ".github/instructions", + ".github/hooks", + "scripts/git-push-approval", + "scripts/tool-guardian", +) + + +def printable(text: str) -> str: + """Escape control characters and '##[' so repository paths cannot start Actions workflow commands.""" + escaped = "".join(character if character.isprintable() else repr(character)[1:-1] for character in text) + return escaped.replace("##[", "#\\x23[") + + +def contains_regular_files(target: Path) -> bool: + pending = [target] + found = False + while pending: + path = pending.pop() + if path.is_symlink(): + raise ValueError(f"Scan content must not contain symlinks: {path}") + mode = path.lstat().st_mode + if stat.S_ISDIR(mode): + pending.extend(path.iterdir()) + elif stat.S_ISREG(mode): + found = True + else: + raise ValueError(f"Scan content must contain only regular files and directories: {path}") + return found + + +def discover_targets(root: Path) -> list[Path]: + apm = root / ".apm" + if not apm.is_dir() or apm.is_symlink(): + raise ValueError("The repository must contain a non-symlink .apm directory.") + + candidates = [] + for base in (root / name for name in CONTENT_ROOTS): + if base.is_symlink(): + raise ValueError(f"Scan targets must not be symlinks: {base}") + if not base.is_dir(): + continue + for path in sorted(base.iterdir()): + if path.is_symlink(): + raise ValueError(f"Scan targets must not be symlinks: {path}") + if path.name == "skills" and path.is_dir(): + candidates.extend(sorted(path.iterdir())) + else: + candidates.append(path) + candidates.extend(root / path for path in EXTRA_ROOTS) + + targets = [] + for path in candidates: + if path.is_symlink() or not path.resolve().is_relative_to(root.resolve()): + raise ValueError(f"Scan targets must stay inside the repository without symlinks: {path}") + if path.exists() and contains_regular_files(path): + targets.append(path) + if not targets: + raise ValueError("No AI primitives were found; refusing an empty scan.") + return targets + + +def relative_parts(path: str) -> tuple[str, ...]: + relative = PurePosixPath(path) + if not path or relative.is_absolute() or ".." in relative.parts or ":" in path or any( + ord(character) < 32 for character in path + ): + raise ValueError("A reported path is outside the selected scan target.") + return relative.parts + + +def repository_path(root: Path, target: Path, filename: str, require_file: bool = True) -> str: + """Map a reported path to a repository path, keeping archive members as 'outer!/member'.""" + if not isinstance(filename, str) or not filename.strip(): + raise ValueError("A finding or diagnostic is missing its file path.") + filename = filename.replace("\\", "/") + if filename.startswith("/scan/"): + filename = filename[len("/scan/"):] + # SkillSpector names inspected archive members '!/', possibly nested. + filename, *members = filename.split("!/") + for member in members: + relative_parts(member) + outer = repository_file(root, target, filename, require_file or bool(members)) + return "!/".join((outer, *members)) + + +def repository_file(root: Path, target: Path, filename: str, require_file: bool) -> str: + parts = relative_parts(filename) + base = target if target.is_dir() else target.parent + candidate = base.joinpath(*parts) + resolved = candidate.resolve() + boundary = target.resolve() + if (target.is_dir() and not resolved.is_relative_to(boundary)) or ( + target.is_file() and resolved != boundary + ): + raise ValueError("A reported path is outside the selected scan target.") + if candidate.is_symlink() or not resolved.is_relative_to(root.resolve()): + raise ValueError("A reported path must remain inside the repository without symlinks.") + if require_file and not candidate.is_file(): + raise ValueError(f"A finding refers to an unavailable source file: {filename}") + return resolved.relative_to(root.resolve()).as_posix() + + +def text_field(record: dict, key: str, default: str = "") -> str: + value = record.get(key) + if value is None: + return default + if not isinstance(value, str): + raise ValueError(f"Expected text in '{key}'.") + return "".join(character for character in value if ord(character) >= 32 or character in "\n\t") + + +def source_line(value: object, optional: bool = False) -> int | None: + if optional and value is None: + return None + if type(value) is not int or value < 1: + raise ValueError("A reported source line must be a positive integer.") + return value + + +def normalize_finding(issue: dict, root: Path, target: Path) -> dict: + if not isinstance(issue, dict) or issue.get("severity") not in SEVERITIES: + raise ValueError("A finding has a missing or unknown severity.") + rule = text_field(issue, "id") + if not rule.strip(): + raise ValueError("A finding has no rule ID.") + location = issue.get("location") + if not isinstance(location, dict): + raise ValueError("A finding has no source location.") + line = source_line(location.get("start_line")) + end_line = source_line(location.get("end_line"), optional=True) + if end_line is not None and end_line < line: + raise ValueError("A finding has an invalid source range.") + fingerprint = issue.get("match_fingerprint") + if fingerprint is not None and not is_fingerprint(fingerprint): + raise ValueError("A finding has an invalid match fingerprint.") + return { + "rule": rule, + "severity": issue["severity"], + "path": repository_path(root, target, location.get("file")), + "line": line, + "end_line": end_line, + "title": text_field(issue, "pattern", rule), + "evidence": text_field(issue, "finding"), + "fingerprint": fingerprint, + "explanation": text_field(issue, "explanation"), + "remediation": text_field(issue, "remediation"), + "accepted": None, + } + + +def normalize_diagnostic(entry: object, field: str, root: Path, target: Path) -> dict: + if not isinstance(entry, dict): + raise ValueError("Expected an analysis diagnostic object.") + filename = entry.get("path") + line = entry.get("start_line") + parser = entry.get("parser_diagnostic") + # Parser limits report their span only inside parser_diagnostic. + if line is None and isinstance(parser, dict): + line = parser.get("start_line") + analyzers = entry.get("analyzers", []) + if not isinstance(analyzers, list) or not all(isinstance(item, str) for item in analyzers): + raise ValueError("Expected a list of analyzer IDs.") + return { + "kind": field, + "reason_code": text_field(entry, "reason_code", field), + "message": text_field(entry, "message"), + "path": repository_path(root, target, filename, require_file=False) if filename else None, + "line": source_line(line, optional=True), + "analyzers": analyzers, + "accepted": None, + } + + +def analyzer_gaps(statuses: object) -> tuple[list[str], list[str]]: + """Return (degraded analyzers that need an accepted limitation, contract errors).""" + if not isinstance(statuses, list): + return [], ["SkillSpector did not report analyzer statuses."] + gaps, errors = [], [] + ran = False + for status in statuses: + if not isinstance(status, dict) or not isinstance(status.get("analyzer_id"), str): + errors.append("Invalid analyzer status entry.") + continue + name, state = status["analyzer_id"], status.get("status") + lost = [key for key in ("failed", "skipped", "unaccounted") if status.get(key, 0) != 0] + # --no-llm disables only the semantic and meta analyzers. + llm = name.startswith("semantic_") or name == "meta_analyzer" + ran = ran or (not llm and state in ("completed", "degraded")) + if lost or not (state in ("completed", "not_applicable", "degraded") or (state == "disabled" and llm)): + errors.append(f"Analyzer {name} did not run to completion ({state}); inspect the JSON report.") + elif state == "degraded": + gaps.append(name) + if not ran: + errors.append("No static analyzer reported running; inspect the JSON report.") + return gaps, errors + + +def read_report(path: Path, root: Path, target: Path) -> dict: + report = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(report, dict): + raise ValueError("Expected a SkillSpector JSON object.") + findings = [] + diagnostics = [] + errors = [] + issues = report.get("issues") + if not isinstance(issues, list): + errors.append("Expected SkillSpector's 'issues' array.") + else: + for index, issue in enumerate(issues, start=1): + try: + findings.append(normalize_finding(issue, root, target)) + except ValueError as error: + errors.append(f"Finding {index} could not be published: {error}") + counts = {severity: sum(item["severity"] == severity for item in findings) for severity in SEVERITIES} + + # Keep usable evidence even when inspection or report validation fails. + if report.get("execution_successful") is not True: + errors.append("SkillSpector did not report successful execution.") + components = report.get("components") + if not isinstance(components, list) or not components: + errors.append("SkillSpector did not report any inspected components.") + else: + for component in components: + try: + if not isinstance(component, dict): + raise ValueError("Expected a component object.") + repository_path(root, target, component.get("path")) + except ValueError as error: + errors.append(f"Invalid inspected component: {error}") + completeness = report.get("analysis_completeness") + if not isinstance(completeness, dict): + completeness = {} + errors.append("SkillSpector did not report analysis completeness.") + if not isinstance(completeness.get("is_complete"), bool): + errors.append("SkillSpector did not report whether inspection was complete.") + gaps, gap_errors = analyzer_gaps(completeness.get("analyzer_statuses")) + errors.extend(gap_errors) + # Upstream completeness excludes out-of-scope files from its denominator. + if completeness.get("scope_exclusions") != []: + errors.append("Inspection has missing or nonempty scope exclusions; inspect the JSON report.") + for field in ("ledger_exceptions", "scope_exclusions"): + entries = completeness.get(field, []) + if not isinstance(entries, list): + errors.append(f"Invalid analysis diagnostic list: {field}") + continue + for entry in entries: + try: + diagnostics.append(normalize_diagnostic(entry, field, root, target)) + except ValueError as error: + errors.append(f"Invalid analysis diagnostic: {error}") + metadata = report.get("metadata") + if not isinstance(metadata, dict) or metadata.get("llm_requested") is not False: + errors.append("The report does not confirm static-only analysis.") + if report.get("suppressed_count") != 0 or report.get("suppressed") != []: + errors.append("Suppressed findings are not permitted by this gate.") + version = metadata.get("skillspector_version") if isinstance(metadata, dict) else None + return { + "counts": counts, + "findings": findings, + "diagnostics": diagnostics, + "complete": completeness.get("is_complete") is True, + "analyzer_gaps": gaps, + "errors": errors, + "version": version if isinstance(version, str) else None, + } + + +def load_accepted(path: Path) -> dict: + """Load owner-reviewed acceptances; a missing file accepts nothing.""" + if not path.exists(): + return {"findings": [], "limitations": []} + data = json.loads(path.read_text(encoding="utf-8")) + schema = { + "findings": ({"path", "rule", "severity", "evidence", "reason"}, {"fingerprint"}), + "limitations": ({"path", "reason_code", "reason"}, {"analyzers"}), + } + if not isinstance(data, dict) or not set(data) <= set(schema): + raise ValueError(f"{path.name} must be an object with 'findings' and 'limitations' lists.") + accepted = {} + for kind, (required, optional) in schema.items(): + entries = data.get(kind, []) + if not isinstance(entries, list): + raise ValueError(f"{path.name}: '{kind}' must be a list.") + for entry in entries: + if not isinstance(entry, dict) or not required <= set(entry) <= required | optional or not all( + isinstance(entry[key], str) and entry[key].strip() for key in required + ): + raise ValueError(f"{path.name}: each '{kind}' entry needs non-empty {sorted(required)}.") + analyzers = entry.get("analyzers", []) + if not isinstance(analyzers, list) or not all(isinstance(item, str) and item for item in analyzers): + raise ValueError(f"{path.name}: 'analyzers' must be a list of analyzer IDs.") + if kind == "findings" and entry["severity"] not in BLOCKING: + raise ValueError(f"{path.name}: only HIGH or CRITICAL findings need acceptance.") + if "fingerprint" in entry and not is_fingerprint(entry["fingerprint"]): + raise ValueError(f"{path.name}: 'fingerprint' must be a SkillSpector match_fingerprint.") + accepted[kind] = entries + return accepted + + +def is_fingerprint(value: object) -> bool: + return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) is not None + + +def finding_matches(entry: dict, finding: dict) -> bool: + # 'evidence' can be a 200-character preview; the fingerprint covers the full match when present. + return entry.get("fingerprint") == finding["fingerprint"] and all( + entry[key] == finding[key] for key in ("path", "rule", "severity", "evidence") + ) + + +def limitation_matches(entry: dict, diagnostic: dict) -> bool: + # The analyzer set is part of the reviewed limitation: a newly degraded analyzer needs review. + return diagnostic["kind"] == "ledger_exceptions" and ( + entry["path"], entry["reason_code"], sorted(entry.get("analyzers", [])) + ) == (diagnostic["path"], diagnostic["reason_code"], sorted(diagnostic["analyzers"])) + + +def apply_acceptances(rows: list[dict], accepted: dict) -> list[str]: + """Mark reviewed findings and limitations; return entries that matched nothing. + + Each entry accepts exactly one occurrence, so a repeated finding needs a repeated entry. + """ + used = set() + + def consume(kind: str, item: dict, matches) -> None: + for index, entry in enumerate(accepted[kind]): + if (kind, index) not in used and matches(entry, item): + item["accepted"] = entry["reason"] + used.add((kind, index)) + return + + for row in rows: + for finding in row["findings"]: + if finding["severity"] in BLOCKING: + consume("findings", finding, finding_matches) + if row["complete"] and not row["analyzer_gaps"]: + continue + for diagnostic in row["diagnostics"]: + consume("limitations", diagnostic, limitation_matches) + return [ + f"Stale {kind} entry for {entry['path']} ({entry.get('rule') or entry.get('reason_code')}); " + "remove it or update it to the current report." + for kind in ("findings", "limitations") + for index, entry in enumerate(accepted[kind]) + if (kind, index) not in used + ] + + +def row_status(row: dict) -> str: + if row["counts"] is None or row["errors"]: + return "ERROR" + if not row["complete"] or row["analyzer_gaps"]: + diagnostics = [item for item in row["diagnostics"] if item["kind"] == "ledger_exceptions"] + covered = {name for item in diagnostics if item["accepted"] for name in item["analyzers"]} + if not diagnostics or any(item["accepted"] is None for item in diagnostics): + row["errors"].append("Static inspection was incomplete; accept or resolve each limitation.") + return "ERROR" + # An accepted limitation only waives the analyzers it names. + if not set(row["analyzer_gaps"]) <= covered: + missing = ", ".join(sorted(set(row["analyzer_gaps"]) - covered)) + row["errors"].append(f"Degraded analyzers have no accepted limitation: {missing}.") + return "ERROR" + blocking = [item for item in row["findings"] if item["severity"] in BLOCKING and not item["accepted"]] + return "BLOCK" if blocking else "PASS" + + +def scan_command(target: Path, output: Path, image: str) -> list[str]: + if any(character in str(path) for path in (target, output) for character in ",\r\n"): + raise ValueError("Docker mount paths must not contain commas or line breaks.") + if os.getuid() == 0: + raise ValueError("Refusing to run the scanner container as root; run as an unprivileged user.") + destination = "/scan" if target.is_dir() else f"/scan/{target.name}" + return [ + "docker", "run", "--rm", "--init", + "--network", "none", + "--read-only", + "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", + "--pids-limit", "128", + "--memory", "2g", + "--cpus", "2", + "--user", f"{os.getuid()}:{os.getgid()}", + "--tmpfs", "/tmp:rw,noexec,nosuid,size=512m", + "--env", "HOME=/tmp", + "--env", "LANGSMITH_TRACING=false", + "--env", "LANGCHAIN_TRACING_V2=false", + "--env", "SKILLSPECTOR_OSV_TIMEOUT=1", + "--mount", f"type=bind,source={target},target={destination},readonly", + "--mount", f"type=bind,source={output},target=/reports", + "--workdir", "/tmp", + "--entrypoint", "timeout", + image, "--kill-after=10s", SCAN_TIMEOUT, + "skillspector", "scan", destination, + "--no-llm", "--fail-on-incomplete", + "--format", "json", "--output", "/reports/report.json", + ] + + +def scan_target(root: Path, target: Path, output: Path, image: str) -> dict: + row = { + "target": target.relative_to(root).as_posix(), "report_directory": output.name, + "counts": None, "findings": [], "diagnostics": [], "complete": False, "analyzer_gaps": [], + "errors": [], "version": None, + } + try: + command = scan_command(target, output, image) + # Keep scanner-controlled text out of the Actions command channel. + with (output / "scanner.log").open("w", encoding="utf-8") as log: + # The in-container timeout can't stop a stalled Docker client. + result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, check=False, + timeout=CLIENT_TIMEOUT) + if result.returncode not in (0, 1): + row["errors"].append(f"Scanner exited {result.returncode}; inspect scanner.log.") + # Exit 1 also covers upstream's aggregate risk score, not this severity policy. + report_path = output / "report.json" + if report_path.is_file() or result.returncode in (0, 1): + parsed = read_report(report_path, root, target) + row["errors"].extend(parsed.pop("errors")) + row.update(parsed) + except subprocess.TimeoutExpired: + row["errors"].append(f"The Docker client did not finish within {CLIENT_TIMEOUT} seconds.") + except (OSError, ValueError) as error: + row["errors"].append(str(error)) + return row + + +def gate_exit_code(rows: list[dict], errors: list[dict]) -> int: + return 2 if errors else int(any(row["status"] == "BLOCK" for row in rows)) + + +def run_scans(root: Path, output: Path, image: str, accepted_path: Path | None = None) -> int: + targets = discover_targets(root) + accepted_path = accepted_path or root / ACCEPTED_FILE + accepted = load_accepted(accepted_path) + for target in targets: + if output == target or target in output.parents: + raise ValueError("Reports must be stored outside all scan targets.") + output.mkdir(parents=True, exist_ok=False) + rows = [] + for index, target in enumerate(targets, start=1): + scope_output = output / f"{index:03d}" + scope_output.mkdir() + rows.append(scan_target(root, target, scope_output, image)) + + stale = apply_acceptances(rows, accepted) + errors = [] + for row in rows: + row["status"] = row_status(row) + if row["errors"]: + errors.append({"target": row["target"], "error": " ".join(row.pop("errors"))}) + else: + row.pop("errors") + accepted_name = accepted_path.relative_to(root).as_posix() if accepted_path.is_relative_to(root) else accepted_path.name + errors.extend({"target": accepted_name, "error": message} for message in stale) + exit_code = gate_exit_code(rows, errors) + versions = {row["version"] for row in rows if row["version"]} + summary = { + "skillspector_version": versions.pop() if len(versions) == 1 else None, + "exit_code": exit_code, + "scans": rows, + "errors": errors, + } + (output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + for row in rows: + counts = row["counts"] or {} + print(f"{row['status']:5} {printable(row['target'])} " + " ".join( + f"{severity}={counts.get(severity, '-')}" for severity in SEVERITIES + )) + print(f"Scanned {len(rows)} targets: {sum(row['status'] == 'BLOCK' for row in rows)} blocked, " + f"{len(errors)} errors. Reports: {output}") + return exit_code + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--image", default="skillspector-ci") + parser.add_argument("--accepted", type=Path, help=f"Reviewed acceptances (default: {ACCEPTED_FILE}).") + args = parser.parse_args() + try: + if os.name != "posix": + raise ValueError("Run this scanner on Linux with Docker (or in WSL).") + accepted = args.accepted.resolve() if args.accepted else None + return run_scans(args.root.resolve(), args.output_dir.resolve(), args.image, accepted) + except (OSError, ValueError) as error: + print(f"SkillSpector runner failed: {printable(str(error))}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_skillspector.py b/tests/test_skillspector.py new file mode 100644 index 0000000..3bb42c4 --- /dev/null +++ b/tests/test_skillspector.py @@ -0,0 +1,405 @@ +"""Regression tests for the SkillSpector gate; no Docker or scanner installation required.""" + +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + + +SPEC = importlib.util.spec_from_file_location( + "scan_skills", Path(__file__).resolve().parents[1] / "scripts" / "scan-skills.py" +) +scanner = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(scanner) + + +def report(*severities, complete=True, ledger=(), degraded=()): + statuses = [{"analyzer_id": "static_patterns", "status": "completed"}, + {"analyzer_id": "semantic_security_discovery", "status": "disabled"}] + statuses += [{"analyzer_id": name, "status": "degraded", "partial": 1} for name in degraded] + return { + "execution_successful": True, + "components": [{"path": "SKILL.md", "type": "markdown"}], + "analysis_completeness": { + "is_complete": complete, "scope_exclusions": [], "ledger_exceptions": list(ledger), + "analyzer_statuses": statuses, + }, + "metadata": {"llm_requested": False, "skillspector_version": "2.13.4"}, + "suppressed_count": 0, + "suppressed": [], + "issues": [{ + "id": "R1", + "severity": severity, + "location": {"file": "SKILL.md", "start_line": 1}, + "pattern": "Synthetic rule", + "explanation": "Synthetic explanation", + "finding": "Synthetic evidence", + "remediation": "Review the finding.", + } for severity in severities], + } + + +PARSE_LIMIT = { + "reason_code": "static_parse_limit", "message": "Bounded parser limit.", "path": "SKILL.md", + "start_line": None, "parser_diagnostic": {"start_line": 1}, "analyzers": ["tool_misuse"], +} +SKILL = ".apm/skills/example/SKILL.md" + + +class ScanTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) / "repo" + self.root.mkdir() + self.output = Path(self.temporary.name) / "reports" + self.skill = self.write(SKILL).parent + self.addCleanup(patch.stopall) + patch.object(os, "getuid", return_value=1000, create=True).start() + patch.object(os, "getgid", return_value=1000, create=True).start() + patch("builtins.print").start() + + def write(self, path, content="Example content\n"): + target = self.root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return target + + def accept(self, findings=(), limitations=()): + self.write(scanner.ACCEPTED_FILE, json.dumps( + {"findings": list(findings), "limitations": list(limitations)} + )) + + def fake_run(self, payload, code=0): + def execute(command, **kwargs): + mount = next(value for value in command if value.endswith(",target=/reports")) + output = Path(mount.removeprefix("type=bind,source=").removesuffix(",target=/reports")) + if payload is not None: + (output / "report.json").write_text(json.dumps(payload), encoding="utf-8") + return subprocess.CompletedProcess(command, code) + return execute + + def run_with(self, payload, code=0): + with patch.object(scanner.subprocess, "run", side_effect=self.fake_run(payload, code)): + return scanner.run_scans(self.root, self.output, "test-scanner") + + def summary(self): + return json.loads((self.output / "summary.json").read_text(encoding="utf-8")) + + def test_target_scope_includes_skills_primitives_agents_and_hooks(self): + for path in ( + ".apm/skills/example/references/context.md", ".apm/skills/second/SKILL.md", + ".apm/prompts/example.prompt.md", ".apm/hooks/example.json", + ".agents/skills/local/SKILL.md", ".agents/notes.md", + ".github/agents/bundled.agent.md", ".github/instructions/x.instructions.md", + ".github/hooks/hook.json", "scripts/tool-guardian/guard-tool.sh", + ".skillspector-source/skills/example/SKILL.md", "README.md", + ): + self.write(path) + targets = [path.relative_to(self.root).as_posix() for path in scanner.discover_targets(self.root)] + self.assertEqual(set(targets), { + ".apm/skills/example", ".apm/skills/second", ".apm/prompts", ".apm/hooks", + ".agents/skills/local", ".agents/notes.md", ".github/agents", + ".github/instructions", ".github/hooks", "scripts/tool-guardian", + }) + + def test_empty_or_missing_apm_is_an_error(self): + empty = self.root / "empty" + empty.mkdir() + with self.assertRaises(ValueError): + scanner.discover_targets(empty) + (empty / ".apm").mkdir() + with self.assertRaises(ValueError): + scanner.discover_targets(empty) + + def test_container_has_no_network_or_credentials_and_bounded_resources(self): + command = scanner.scan_command(self.skill, self.output, "test-scanner") + self.assertEqual(command[command.index("--network") + 1], "none") + self.assertEqual(command[command.index("--user") + 1], "1000:1000") + for value in ("--read-only", "--no-llm", "--fail-on-incomplete", "--pids-limit", + "--memory", "--cpus", "--kill-after=10s", "180s"): + self.assertIn(value, command) + self.assertIn(f"type=bind,source={self.skill},target=/scan,readonly", command) + for value in ("--transitive", "--baseline", "--use-shipped-baseline", "--env-file"): + self.assertNotIn(value, command) + + def test_single_files_keep_their_extension(self): + target = self.write(".apm/example.md") + command = scanner.scan_command(target, self.output, "test-scanner") + self.assertIn(f"type=bind,source={target},target=/scan/example.md,readonly", command) + + def test_mount_path_cannot_inject_docker_options(self): + target = self.write(".apm/skills/example,source=outside/SKILL.md").parent + with self.assertRaises(ValueError): + scanner.scan_command(target, self.output, "test-scanner") + + def test_root_cannot_run_the_scanner_container(self): + with patch.object(os, "getuid", return_value=0, create=True), self.assertRaisesRegex(ValueError, "root"): + scanner.scan_command(self.skill, self.output, "test-scanner") + + def test_stalled_docker_client_is_an_error(self): + stalled = subprocess.TimeoutExpired("docker", scanner.CLIENT_TIMEOUT) + with patch.object(scanner.subprocess, "run", side_effect=stalled) as run: + self.assertEqual(scanner.run_scans(self.root, self.output, "test-scanner"), 2) + self.assertEqual(run.call_args.kwargs["timeout"], scanner.CLIENT_TIMEOUT) + self.assertIn("did not finish", self.summary()["errors"][0]["error"]) + + def test_symlinks_are_rejected_before_any_scan(self): + self.write(".apm/skills/example/references/context.md") + original = Path.is_symlink + for link in (self.root / ".apm" / "skills", self.skill / "SKILL.md", self.skill / "references"): + with self.subTest(link=link), \ + patch.object(Path, "is_symlink", lambda path, link=link: path == link or original(path)), \ + patch.object(scanner.subprocess, "run") as run: + with self.assertRaisesRegex(ValueError, "symlinks"): + scanner.run_scans(self.root, self.output, "test-scanner") + run.assert_not_called() + + @unittest.skipUnless(os.name == "posix", "Real symlink creation requires a POSIX test host.") + def test_real_descendant_symlinks_are_rejected_without_following(self): + primary = self.skill / "SKILL.md" + primary.unlink() + outside = self.write("outside-scan.md") + for target in (outside, self.root / "missing.md", self.skill): + with self.subTest(target=target): + primary.symlink_to(target, target_is_directory=target.is_dir()) + try: + with self.assertRaisesRegex(ValueError, "symlinks"): + scanner.discover_targets(self.root) + finally: + primary.unlink() + + def test_walk_errors_fail_instead_of_silently_skipping_content(self): + original = Path.iterdir + def iterdir(path): + if path == self.skill: + raise PermissionError("Unreadable skill") + return original(path) + with patch.object(Path, "iterdir", iterdir): + with self.assertRaises(PermissionError): + scanner.discover_targets(self.root) + + def test_high_and_critical_block_even_with_zero_cli_exit(self): + for severity in ("HIGH", "CRITICAL"): + with self.subTest(severity=severity): + self.output = Path(self.temporary.name) / severity + self.assertEqual(self.run_with(report(severity)), 1) + self.assertEqual(self.summary()["exit_code"], 1) + + def test_low_and_medium_pass_even_with_aggregate_risk_exit_one(self): + self.assertEqual(self.run_with(report("LOW", "MEDIUM"), code=1), 0) + + def test_scanner_failures_are_errors(self): + for name, payload, code in (("crash", report(), 2), ("timeout", None, 124), ("missing", None, 0)): + with self.subTest(name): + self.output = Path(self.temporary.name) / name + self.assertEqual(self.run_with(payload, code), 2) + if payload is None: + self.assertIsNone(self.summary()["scans"][0]["counts"]) + + def test_docker_unavailable_is_an_error(self): + with patch.object(scanner.subprocess, "run", side_effect=FileNotFoundError("docker")): + self.assertEqual(scanner.run_scans(self.root, self.output, "test-scanner"), 2) + + def test_report_contract_fails_closed(self): + def completeness(**changes): + return {**report(), "analysis_completeness": {**report()["analysis_completeness"], **changes}} + invalid = [ + [], {}, + {**report(), "components": []}, + {**report(), "components": [None]}, + {**report(), "components": [{"path": "../outside.md"}]}, + {**report(), "components": [{"path": "never-inspected.md"}]}, + {**report(), "issues": None}, + {**report(), "issues": [{"severity": "UNKNOWN"}]}, + {**report(), "execution_successful": False}, + {**report(), "analysis_completeness": None}, + {**report(), "analysis_completeness": {"is_complete": True}}, + completeness(is_complete=None), + completeness(is_complete="true"), + completeness(scope_exclusions=[{"path": "references/", "reason_code": "excluded"}]), + completeness(analyzer_statuses=None), + completeness(analyzer_statuses=[]), + completeness(analyzer_statuses=[{"analyzer_id": "semantic_security_discovery", "status": "disabled"}]), + completeness(analyzer_statuses=[{"analyzer_id": "static_patterns", "status": "not_applicable"}]), + completeness(analyzer_statuses=[{"analyzer_id": "static_patterns", "status": "failed"}]), + completeness(analyzer_statuses=[{"analyzer_id": "static_patterns", "status": "disabled"}]), + completeness(analyzer_statuses=[{"analyzer_id": "x", "status": "completed", "unaccounted": 1}]), + {**report(), "metadata": {"llm_requested": True}}, + {**report(), "suppressed_count": 1}, + ] + for index, payload in enumerate(invalid): + with self.subTest(payload=payload): + self.output = Path(self.temporary.name) / f"invalid-{index}" + self.assertEqual(self.run_with(payload), 2) + + def test_all_targets_are_scanned_after_findings_or_errors(self): + self.write(".apm/skills/second/SKILL.md") + results = iter([(None, 2), (report("CRITICAL"), 1)]) + def execute(command, **kwargs): + payload, code = next(results) + return self.fake_run(payload, code)(command, **kwargs) + with patch.object(scanner.subprocess, "run", side_effect=execute) as run: + self.assertEqual(scanner.run_scans(self.root, self.output, "test-scanner"), 2) + self.assertEqual(run.call_count, 2) + self.assertEqual([row["status"] for row in self.summary()["scans"]], ["ERROR", "BLOCK"]) + + def test_existing_or_nested_report_directories_are_rejected(self): + with self.assertRaises(ValueError): + scanner.run_scans(self.root, self.skill / "reports", "test-scanner") + self.output.mkdir() + with self.assertRaises(FileExistsError): + scanner.run_scans(self.root, self.output, "test-scanner") + + def test_incomplete_scan_retains_findings_and_diagnostic_lines(self): + self.assertEqual(self.run_with(report("HIGH", complete=False, ledger=[PARSE_LIMIT]), code=1), 2) + row = self.summary()["scans"][0] + self.assertEqual((row["status"], row["counts"]["HIGH"]), ("ERROR", 1)) + self.assertEqual(row["findings"][0]["path"], SKILL) + self.assertEqual((row["diagnostics"][0]["path"], row["diagnostics"][0]["line"]), (SKILL, 1)) + self.assertIn("incomplete", self.summary()["errors"][0]["error"]) + + def test_untrusted_locations_and_lines_fail_closed_without_hiding_others(self): + cases = [("file", name) for name in ( + "../outside.md", "/etc/passwd", "C:\\outside.md", "/scan/../x.md", "missing.md", "SKILL.md\nx", + )] + [("start_line", line) for line in (0, -1, True, "1", None)] + for index, (key, value) in enumerate(cases): + with self.subTest(key=key, value=value): + self.output = Path(self.temporary.name) / f"location-{index}" + payload = report("HIGH", "MEDIUM") + payload["issues"][1]["location"][key] = value + self.assertEqual(self.run_with(payload), 2) + self.assertEqual(len(self.summary()["scans"][0]["findings"]), 1) + + def test_file_targets_and_supporting_files_map_to_repository_paths(self): + target = self.write(".apm/example.md") + self.assertEqual(scanner.repository_path(self.root, target, "/scan/example.md"), ".apm/example.md") + with self.assertRaises(ValueError): + scanner.repository_path(self.root, target, "different.md") + self.write(".apm/skills/example/references/context.md") + self.assertEqual( + scanner.repository_path(self.root, self.skill, "/scan/references/context.md"), + ".apm/skills/example/references/context.md", + ) + + def test_inspected_archive_members_map_to_the_outer_archive(self): + self.write(".apm/skills/example/bundle.zip", "zip bytes") + members = ["bundle.zip", "bundle.zip!/payload.sh", "bundle.zip!/inner.zip!/notes.md"] + payload = report("HIGH") + payload["components"] += [{"path": path, "type": "archive"} for path in members] + payload["issues"][0]["location"]["file"] = "bundle.zip!/payload.sh" + self.assertEqual(self.run_with(payload), 1) + finding = self.summary()["scans"][0]["findings"][0] + self.assertEqual(finding["path"], ".apm/skills/example/bundle.zip!/payload.sh") + self.accept([{"path": finding["path"], "rule": "R1", "severity": "HIGH", + "evidence": "Synthetic evidence", "reason": "Reviewed archive member."}]) + self.output = Path(self.temporary.name) / "accepted" + self.assertEqual(self.run_with(payload), 0) + for index, path in enumerate(( + "missing.zip!/payload.sh", "references!/x.md", "bundle.zip!/", "bundle.zip!/../x.md", + "bundle.zip!//etc/passwd", "../bundle.zip!/x.md", + )): + with self.subTest(path=path): + self.output = Path(self.temporary.name) / f"archive-{index}" + self.assertEqual(self.run_with({**report(), "components": [{"path": path}]}), 2) + + def test_reviewed_acceptance_passes_only_the_matching_finding(self): + entry = {"path": SKILL, "rule": "R1", "severity": "HIGH", "evidence": "Synthetic evidence", + "reason": "Reviewed false positive."} + self.accept([entry]) + self.assertEqual(self.run_with(report("HIGH")), 0) + finding = self.summary()["scans"][0]["findings"][0] + self.assertEqual(finding["accepted"], "Reviewed false positive.") + for name, payload in (("critical", report("CRITICAL")), ("changed", report("HIGH"))): + with self.subTest(name): + self.output = Path(self.temporary.name) / name + payload["issues"][0]["finding"] = "Different evidence" if name == "changed" else "Synthetic evidence" + # The stale entry also fails, so a new finding cannot hide behind an old approval. + self.assertEqual(self.run_with(payload), 2) + self.assertEqual(self.summary()["scans"][0]["status"], "BLOCK") + self.assertIn("Stale findings entry", self.summary()["errors"][0]["error"]) + + def test_each_entry_accepts_exactly_one_occurrence(self): + entry = {"path": SKILL, "rule": "R1", "severity": "HIGH", "evidence": "Synthetic evidence", + "reason": "Deny-list."} + self.accept([entry]) + self.assertEqual(self.run_with(report("HIGH", "HIGH")), 1) + self.output = Path(self.temporary.name) / "two-entries" + self.accept([entry, entry]) + self.assertEqual(self.run_with(report("HIGH", "HIGH")), 0) + + def test_accepted_limitations_allow_incomplete_scans(self): + self.accept(limitations=[{"path": SKILL, "reason_code": "static_parse_limit", + "analyzers": ["tool_misuse"], "reason": "Reviewed."}]) + incomplete = {"complete": False, "degraded": ["tool_misuse"]} + self.assertEqual(self.run_with(report(ledger=[PARSE_LIMIT], **incomplete), code=1), 0) + other = {**PARSE_LIMIT, "reason_code": "reference_missing"} + widened = {**PARSE_LIMIT, "analyzers": ["tool_misuse", "taint"]} + for name, payload in ( + ("other", report(ledger=[PARSE_LIMIT, other], **incomplete)), + ("no-ledger", report(complete=False)), + # The accepted limitation names tool_misuse, so it cannot waive an unrelated analyzer. + ("unrelated", report(ledger=[PARSE_LIMIT], complete=False, degraded=["tool_misuse", "taint"])), + # A reviewed limitation that now degrades another analyzer needs a new review. + ("widened", report(ledger=[widened], complete=False, degraded=["tool_misuse", "taint"])), + ): + with self.subTest(name): + self.output = Path(self.temporary.name) / name + self.assertEqual(self.run_with(payload, code=1), 2) + + def test_fingerprint_binds_acceptance_to_the_full_match(self): + preview, original, changed = "x" * 200, "a" * 64, "b" * 64 + entry = {"path": SKILL, "rule": "R1", "severity": "HIGH", "evidence": preview, + "fingerprint": original, "reason": "Reviewed long match."} + def payload(fingerprint): + data = report("HIGH") + data["issues"][0].update(finding=preview, match_fingerprint=fingerprint) + return data + self.accept([entry]) + self.assertEqual(self.run_with(payload(original)), 0) + unbound = {key: value for key, value in entry.items() if key != "fingerprint"} + # Same 200-character preview, different suffix: the old approval must not apply. + for name, data, accepted in (("changed-suffix", payload(changed), entry), + ("unbound-entry", payload(original), unbound), + ("invalid-report", payload("not-a-hash"), entry)): + with self.subTest(name): + self.output = Path(self.temporary.name) / name + self.accept([accepted]) + self.assertEqual(self.run_with(data), 2) + if name != "invalid-report": + self.assertEqual(self.summary()["scans"][0]["status"], "BLOCK") + + def test_printed_paths_cannot_start_workflow_commands(self): + self.assertEqual(scanner.printable("x\n::warning::y\r\x1b"), "x\\n::warning::y\\r\\x1b") + self.assertNotIn("##[", scanner.printable(".apm/skills/##[warning]injected")) + + def test_stale_acceptance_is_an_error(self): + self.accept([{"path": SKILL, "rule": "R9", "severity": "HIGH", "evidence": "x", "reason": "Old."}]) + self.assertEqual(self.run_with(report()), 2) + self.assertEqual(self.summary()["errors"][0]["target"], scanner.ACCEPTED_FILE) + + def test_invalid_acceptance_files_are_rejected(self): + base = {"path": SKILL, "rule": "R1", "severity": "HIGH", "evidence": "e", "reason": "Reviewed."} + limitation = {"path": SKILL, "reason_code": "static_parse_limit", "reason": "Reviewed."} + for index, content in enumerate(( + "[]", '{"other": []}', '{"findings": {}}', + json.dumps({"findings": [{**base, "reason": " "}]}), + json.dumps({"findings": [{**base, "severity": "MEDIUM"}]}), + json.dumps({"findings": [{**base, "line": "1"}]}), + json.dumps({"findings": [{**base, "fingerprint": "ABC"}]}), + json.dumps({"findings": [{key: value for key, value in base.items() if key != "evidence"}]}), + json.dumps({"limitations": [{"path": SKILL, "reason": "No code."}]}), + json.dumps({"limitations": [{**limitation, "analyzers": "tool_misuse"}]}), + )): + with self.subTest(content=content): + self.write(scanner.ACCEPTED_FILE, content) + with self.assertRaises(ValueError): + scanner.run_scans(self.root, Path(self.temporary.name) / f"bad-{index}", "test-scanner") + + +if __name__ == "__main__": + unittest.main()