Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 168 additions & 0 deletions .github/skillspector-accepted.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
{
"findings": [
{
"path": ".apm/skills/obsidian/SKILL.md",
"rule": "AE1",
"severity": "HIGH",
"evidence": "references/FUNCTIONS_REFERENCE.md (partial)",
"fingerprint": "a98fb594e6cdfe645f2a06d85f243bb80e4f5f02cadf94c381d57c0eae8b9624",
"reason": "Derived from the accepted static_parse_limit on FUNCTIONS_REFERENCE.md: a Markdown link to bundled reference docs, not a risky artifact."
},
{
"path": ".apm/skills/renovate-triage-and-merge/references/testing.md",
"rule": "AR2",
"severity": "HIGH",
"evidence": "no warning",
"fingerprint": "449e47b25897541aa26f8824417d1280ef58d2d29e5e3ba95502143b7207c9ec",
"reason": "Test expectation that happy-path results emit no warning; not an instruction to suppress refusals."
},
{
"path": ".apm/skills/renovate-triage-and-merge/references/testing.md",
"rule": "AR2",
"severity": "HIGH",
"evidence": "without warnings",
"fingerprint": "5794522f6c58178944cac8e9c872993a23e09b00e15b2f4981f8cd97acee1cf9",
"reason": "Test expectation that valid ports are accepted without warnings; not an instruction to suppress refusals."
},
{
"path": ".apm/skills/renovate-triage-and-merge/scripts/renovate_prs.py",
"rule": "E2",
"severity": "HIGH",
"evidence": "os.environ.copy()",
"fingerprint": "23b3d476caf63d859a2a3bade37548b7986b00b138b45df1b6a49782c4e2984b",
"reason": "Passes the caller's environment to the local gh CLI so it can authenticate; nothing is read out or sent elsewhere."
},
{
"path": ".github/agents/project-architecture-planner.agent.md",
"rule": "YR4",
"severity": "HIGH",
"evidence": "tools:; description:; <!-- System boundary -->",
"fingerprint": "137048b3b6be085cab625795bafa2f3f257af88a999bbfd670732e79f71093d2",
"reason": "External awesome-copilot agent: the HTML comment labels a draw.io diagram example, not a hidden tool instruction."
},
{
"path": ".github/agents/project-architecture-planner.agent.md",
"rule": "P2",
"severity": "HIGH",
"evidence": "<!-- System boundary -->",
"fingerprint": "ad470b71e9667217b4b56927c5358c7cc28e4e07efdc5a1121b76747386b0e30",
"reason": "External awesome-copilot agent: XML comment inside a draw.io diagram example, not a hidden instruction."
},
{
"path": ".github/agents/sast-sca-security-analyzer.agent.md",
"rule": "TM1",
"severity": "HIGH",
"evidence": "subprocess.call(cmd, shell=True",
"fingerprint": "779fb9591a6b32e605e7608dc5d7984b91a8bc98090ffd38127fc5b254509836",
"reason": "External awesome-copilot agent: listed as a vulnerable pattern to detect (CWE-78), not code to run."
},
{
"path": "scripts/tool-guardian/guard-tool.sh",
"rule": "TM1",
"severity": "HIGH",
"evidence": "rm -rf /",
"fingerprint": "f3ff4d6cfbe1b7f6b23dd4d1379edae984c14738c5f5b14db40e215beb673f08",
"reason": "Tool Guardian's deny-list: these destructive commands are the regexes it blocks, never commands it runs."
},
{
"path": "scripts/tool-guardian/guard-tool.sh",
"rule": "TM1",
"severity": "HIGH",
"evidence": "'rm' on specific paths instead of root\"\n \"destructive_file_ops:::critical:::rm -rf ~:::Use targeted 'rm' on specific paths instead of home directory\"\n \"destructive_file_ops:::critical:::rm -rf \\",
"fingerprint": "7fe134484b49b5a987f3c2e2a509e41683442354c0696b3fef87bbbbd234da61",
"reason": "Tool Guardian's deny-list: these destructive commands are the regexes it blocks, never commands it runs."
},
{
"path": "scripts/tool-guardian/guard-tool.sh",
"rule": "TM1",
"severity": "HIGH",
"evidence": "rm -rf /:::Use",
"fingerprint": "1e47dfe1c665fe0312ed8554496c3bfd7db7e46c694b1bd9f361bc38e22f8b4d",
"reason": "Tool Guardian's deny-list: these destructive commands are the regexes it blocks, never commands it runs."
},
{
"path": "scripts/tool-guardian/guard-tool.sh",
"rule": "TM1",
"severity": "HIGH",
"evidence": "rm -rf ~",
"fingerprint": "26e22f4cfd2aa77cd384519bed5efdd6f6dba6aab2e61d67ae29c1cbcadc466e",
"reason": "Tool Guardian's deny-list: these destructive commands are the regexes it blocks, never commands it runs."
},
{
"path": "scripts/tool-guardian/guard-tool.sh",
"rule": "PE3",
"severity": "HIGH",
"evidence": ".env ",
"fingerprint": "66ef81c27c03406fd9935e5fc86fc4126d7150c15b8d076fdc528d05cea7c7bf",
"reason": "Tool Guardian's deny-list text about protecting .env files; it does not read credentials."
},
{
"path": "scripts/tool-guardian/guard-tool.sh",
"rule": "TM1",
"severity": "HIGH",
"evidence": "git push --force",
"fingerprint": "588985d404a57a20e57a9346ad923a32f942548b83c815a3d0565b5856c6609c",
"reason": "Tool Guardian's deny-list: these destructive commands are the regexes it blocks, never commands it runs."
},
{
"path": "scripts/tool-guardian/guard-tool.sh",
"rule": "TM1",
"severity": "HIGH",
"evidence": "git push --force",
"fingerprint": "588985d404a57a20e57a9346ad923a32f942548b83c815a3d0565b5856c6609c",
"reason": "Tool Guardian's deny-list: these destructive commands are the regexes it blocks, never commands it runs."
},
{
"path": "scripts/tool-guardian/guard-tool.sh",
"rule": "TM1",
"severity": "HIGH",
"evidence": "git reset --hard",
"fingerprint": "f21f5124192a6ca491d13034d2b1dcaae69642decbd659201773b11a2bc5c2a5",
"reason": "Tool Guardian's deny-list: these destructive commands are the regexes it blocks, never commands it runs."
},
{
"path": "scripts/tool-guardian/guard-tool.sh",
"rule": "TM1",
"severity": "HIGH",
"evidence": "chmod 777:::Use 'chmod 755' for directories or 'chmod 644' for files\"\n \"permission_abuse:::high:::chmod -R 777",
"fingerprint": "2df784ea404c1b90428a92adcdca4a3baa68d299c8051bba0d1f995439b20c8d",
"reason": "Tool Guardian's deny-list: these destructive commands are the regexes it blocks, never commands it runs."
}
],
"limitations": [
{
"path": ".apm/skills/obsidian/SKILL.md",
"reason_code": "reference_missing",
"reason": "The path-like reference is generic `[text](url)` link syntax in prose, not a bundled file."
},
{
"path": ".apm/skills/obsidian/references/FUNCTIONS_REFERENCE.md",
"reason_code": "static_parse_limit",
"analyzers": [
"static_patterns_tool_misuse"
],
"reason": "Obsidian Bases formula examples in a reference doc; the parser cannot bound them as commands but checked all input."
},
{
"path": ".apm/skills/tripit-exporter/SKILL.md",
"reason_code": "static_parse_limit",
"analyzers": [
"static_patterns_tool_misuse"
],
"reason": "A JavaScript template literal builds an output file name at runtime; the export only writes a local JSON file."
},
{
"path": ".apm/skills/tripit-exporter/SKILL.md",
"reason_code": "reference_missing",
"reason": "Links to the repository-root THIRD_PARTY_NOTICES.md, which lives outside the skill directory by design."
},
{
"path": ".github/agents/sast-sca-security-analyzer.agent.md",
"reason_code": "static_parse_limit",
"analyzers": [
"static_patterns_tool_misuse"
],
"reason": "External awesome-copilot agent: example report/code snippets the parser cannot bound; all parser input was checked."
}
]
}
4 changes: 2 additions & 2 deletions .github/workflows/markdown-validation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ jobs:
- name: Check vendored primitives keep their local overrides
run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_overrides.py' -v

- name: Test SkillSpector severity gate
run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_skillspector.py' -v
- name: Test SkillSpector gate and report
run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_skillspector*.py' -v

consumer-install:
name: Check Consumer Install
Expand Down
149 changes: 149 additions & 0 deletions .github/workflows/skillspector.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
---
name: SkillSpector

# Static NVIDIA SkillSpector scan of every AI primitive. "Scan AI primitives"
# reports on every PR so it can be a required check; it only builds and scans
# when the PR touches AI primitives or the scanner (scripts/skillspector-changes.sh).
on:
pull_request:
push:
branches:
- main
schedule:
- cron: "23 5 * * 1"
workflow_dispatch:

permissions:
contents: read

concurrency:
group: skillspector-${{ github.ref }}
cancel-in-progress: true

jobs:
scan:
name: Scan AI primitives
runs-on: ubuntu-24.04
# Targets scan sequentially with a 300 s worst case each (CLIENT_TIMEOUT); keep room for
# every target to time out (about 20 today) plus build, smoke tests and publishing.
# A normal run takes about 3 minutes.
timeout-minutes: 150
permissions:
contents: read
security-events: write
outputs:
scanned: ${{ steps.changes.outputs.relevant }}
steps:
- name: Harden runner
uses: DevSecNinja/.github/actions/harden-runner@27fd66740c629ea38380dcd3bf93e7bc74c54247 # v1.9.0

- name: Checkout toolkit
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 2
persist-credentials: false

- name: Check for AI primitive changes
id: changes
run: bash scripts/skillspector-changes.sh

- name: Test scan gate and report
if: steps.changes.outputs.relevant == 'true'
run: python3 -m unittest discover -s tests -p 'test_skillspector*.py'

- name: Checkout pinned SkillSpector
if: steps.changes.outputs.relevant == 'true'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: NVIDIA/SkillSpector
# renovate: datasource=github-tags depName=NVIDIA/SkillSpector
ref: 54824463814a0be01360792523ec7245a69190fc # v2.13.4
path: .skillspector-source
persist-credentials: false

- name: Build scanner
if: steps.changes.outputs.relevant == 'true'
run: docker build --tag skillspector-ci .skillspector-source

- name: Exercise the real scanner container
if: steps.changes.outputs.relevant == 'true'
run: python3 tests/smoke-skillspector.py --output-dir "$RUNNER_TEMP/skillspector-reports/smoke"

- name: Scan and enforce severity gate
id: scan
if: steps.changes.outputs.relevant == 'true'
run: python3 scripts/scan-skills.py --output-dir "$RUNNER_TEMP/skillspector-reports/repository"

- name: Render summary and SARIF
id: report
if: ${{ !cancelled() && steps.scan.outcome != 'skipped' }}
run: >-
python3 scripts/skillspector-report.py "$RUNNER_TEMP/skillspector-reports/repository/summary.json"
--markdown "$GITHUB_STEP_SUMMARY" --sarif "$RUNNER_TEMP/skillspector-reports/repository/findings.sarif"

- name: Publish findings to Code Scanning
if: ${{ !cancelled() && steps.report.outcome == 'success' }}
uses: github/codeql-action/upload-sarif@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3
with:
sarif_file: ${{ runner.temp }}/skillspector-reports/repository/findings.sarif
category: skillspector/static
wait-for-processing: true

- name: Upload reports
if: ${{ !cancelled() && steps.changes.outputs.relevant == 'true' }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: skillspector-reports
path: ${{ runner.temp }}/skillspector-reports
if-no-files-found: warn
retention-days: 14

comment:
name: Post SkillSpector report
needs: scan
# Always reports on PRs: a skipped job blocks tools that require every check to succeed.
if: ${{ !cancelled() && github.event_name == 'pull_request' }}
runs-on: ubuntu-24.04
timeout-minutes: 5
permissions:
contents: read
pull-requests: write
env:
# Only same-repository PRs that were scanned get a comment; fork PRs have a read-only token.
POST: ${{ needs.scan.outputs.scanned == 'true' && !github.event.pull_request.head.repo.fork }}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
# A PR that stopped changing AI primitives marks its earlier comment as stale instead.
CLEAR: ${{ needs.scan.outputs.scanned == 'false' && !github.event.pull_request.head.repo.fork }}
steps:
- name: Harden runner
uses: DevSecNinja/.github/actions/harden-runner@27fd66740c629ea38380dcd3bf93e7bc74c54247 # v1.9.0

- name: Checkout toolkit
if: env.POST == 'true' || env.CLEAR == 'true'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: Download reports
if: env.POST == 'true'
# A missing artifact still produces a "no readable results" comment below.
continue-on-error: true
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: skillspector-reports
path: ${{ runner.temp }}/skillspector-reports

- name: Create or update the PR comment
if: env.POST == 'true'
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: >-
python3 scripts/skillspector-report.py "$RUNNER_TEMP/skillspector-reports/repository/summary.json"
--comment-pr "$PR_NUMBER"

- name: Mark an earlier PR comment as not needed
if: env.CLEAR == 'true'
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: python3 scripts/skillspector-report.py --not-needed-pr "$PR_NUMBER"
4 changes: 2 additions & 2 deletions .lefthook.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_
[pre-commit.commands.apm-overrides]
run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_overrides.py'"

# SkillSpector gate: severity policy, report contract and reviewed acceptances.
# SkillSpector gate and report: severity policy, reviewed acceptances, PR comment.
[pre-commit.commands.skillspector-gate]
run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_skillspector.py'"
run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_skillspector*.py'"

# Regenerate INDEX.md / README index from .apm/ and re-stage the result, so the
# generated index never drifts from the primitives (the common contributor miss).
Expand Down
Loading
Loading