diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5008a10..64afbfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,3 +94,19 @@ jobs: if poetry run aisbom scan hf://google-bert/bert-base-uncased --offline --no-fail-on-risk; then echo "::error::--offline scanned a remote target"; exit 1 fi + + # Both frameworks, one local and one remote target, and every file each + # pack promises — so a pack that silently dropped a document fails here. + - name: Smoke Test #14 - Documentation Packs (comply) + run: | + poetry run aisbom comply hf://google-bert/bert-base-uncased --framework eu-ai-act --output-dir comply-eu + test -s comply-eu/annex-xi-technical-documentation.md + test -s comply-eu/art53-training-content-summary.md + test -s comply-eu/sbom.json + test -s comply-eu/README.md + poetry run aisbom comply demo_data --framework fda --output-dir comply-fda + test -s comply-fda/sbom.json + test -s comply-fda/sbom.openvex.json + test -s comply-fda/sbom.vex.cdx.json + test -s comply-fda/component-support.csv + test -s comply-fda/README.md diff --git a/README.md b/README.md index 52fe606..05f2630 100644 --- a/README.md +++ b/README.md @@ -484,6 +484,34 @@ per-document denominator is not comparable with anyone else's. `aisbom score` grades CycloneDX. SPDX input is rejected rather than partially scored. +### Documentation packs (`aisbom comply`) + +`aisbom comply` scans a target and writes a folder of regulator-facing +documents pre-filled from what the scan read: + +```bash +aisbom comply hf://google-bert/bert-base-uncased --framework eu-ai-act +aisbom comply ./models --framework fda --output-dir ./submission/sbom +``` + +| `--framework` | Writes | +|---|---| +| `eu-ai-act` | `annex-xi-technical-documentation.md` (Annex XI skeleton), `art53-training-content-summary.md` (in the sections of the Commission's training-content template), `sbom.json`, `README.md` | +| `fda` | `sbom.json`, `sbom.openvex.json`, `sbom.vex.cdx.json`, `component-support.csv` (support level and end-of-support date per component), `README.md` | + +A field is filled only from something the scan read, such as the model card's +task, architecture, licence and declared datasets, file formats, hashes and +security findings. Each value names its source. Everything else, including +provider identity, release dates, training process, compute and lifecycle +dates, is left as `TODO (provider)` rather than guessed. +[docs/comply.md](docs/comply.md) lists every field and where its value comes +from. + +The pack describes files, not your organisation, and is not legal advice. +A scan with an unreadable target exits `1` and writes nothing. Otherwise the +command exits `0` whatever the findings; gate CI with `aisbom scan`. `--offline`, +`--no-osv` and `--no-hf-lookup` work as they do for `scan`. + --- ## Use as a GitHub Action @@ -696,6 +724,10 @@ If you explicitly use `--share`: the generated `sbom.json` document is uploaded Per `aisbom diff`: a `cli_diff` event with `has_drift=true|false`. +Per `aisbom score`: a `cli_score` event with the letter `grade` and the number of `models`. + +Per `aisbom comply`: a `cli_comply` event with the `framework` (`eu-ai-act` or `fda`). The generated documents are never sent. + On a scan fetch failure or unhandled exception: a `cli_error` event records the exception class name only (e.g. `JSONDecodeError`), a low-cardinality `http_status` bucket (e.g. `401`, `timeout`), a `token_present` boolean (whether an `HF_TOKEN` / `HUGGING_FACE_HUB_TOKEN` was set), and a `consecutive_failures` bucket (`1`–`9` or `10+` — how many runs in a row hit the same failure shape) — never the token value, the message, the traceback, a URL, or any file content. When the scan target itself is unusable (a path that doesn't exist, a file no scanner supports, or something that isn't a regular file or directory): a `cli_error` event with the same fields, where `error_type` is one of a fixed set — `MissingTarget`, `UnsupportedFileType`, or `NotAFileOrDirectory` — and `http_status` is `none`. Never the path, the file name, or its extension. @@ -706,17 +738,17 @@ To detect failure loops (see [Authentication](#authentication-private--gated-hug Each event carries an anonymous `user_id` — a SHA-256 of your machine's MAC address plus an app salt, truncated to 16 hex chars. Stored in `~/.aisbom/config.json`. Lets us see returning users without identifying anyone. -### OSV lookups (`--vex` only) +### OSV lookups (`--vex` and `comply --framework fda` only) -When you pass `--vex` and the scan finds exact `requirements.txt` pins, AIsbom sends each pinned **package name and version** to the public OSV API at `https://api.osv.dev`, and fetches the advisories it names. Nothing else is sent: no file paths, model names, hashes, findings, or identifiers. This is a request to a third-party service, not telemetry, so `AISBOM_NO_TELEMETRY` does not affect it; `--no-osv`, `AISBOM_NO_OSV=1` or `--offline` does. Responses are cached locally in `~/.aisbom/osv_cache.json` for 24 hours, and deleting that file is always safe. +When you pass `--vex`, or run `aisbom comply --framework fda`, and the scan finds exact `requirements.txt` pins, AIsbom sends each pinned **package name and version** to the public OSV API at `https://api.osv.dev`, and fetches the advisories it names. Nothing else is sent: no file paths, model names, hashes, findings, or identifiers. This is a request to a third-party service, not telemetry, so `AISBOM_NO_TELEMETRY` does not affect it; `--no-osv`, `AISBOM_NO_OSV=1` or `--offline` does. Responses are cached locally in `~/.aisbom/osv_cache.json` for 24 hours, and deleting that file is always safe. ### PyPI license lookups (on by default) -When a scan finds exact `requirements.txt` pins and writes CycloneDX or SPDX 2.3 output, AIsbom requests each pinned **package name and version** from the public PyPI JSON API at `https://pypi.org/pypi///json` to read its declared license. Nothing else is sent: no file paths, model names, hashes, findings, or identifiers. Like the OSV lookup this is a request to a third-party service, not telemetry, so `AISBOM_NO_TELEMETRY` does not affect it; `--offline` or `AISBOM_OFFLINE=1` does. Answers are cached locally in `~/.aisbom/pypi_license_cache.json`, and deleting that file is always safe. +When a scan finds exact `requirements.txt` pins and writes CycloneDX or SPDX 2.3 output (including the SBOM `aisbom score` and `aisbom comply` build), AIsbom requests each pinned **package name and version** from the public PyPI JSON API at `https://pypi.org/pypi///json` to read its declared license. Nothing else is sent: no file paths, model names, hashes, findings, or identifiers. Like the OSV lookup this is a request to a third-party service, not telemetry, so `AISBOM_NO_TELEMETRY` does not affect it; `--offline` or `AISBOM_OFFLINE=1` does. Answers are cached locally in `~/.aisbom/pypi_license_cache.json`, and deleting that file is always safe. ### Hugging Face model-card lookups (local scans) -When a local scan writes CycloneDX 1.7 or SPDX 3.0 output (or `aisbom score` grades a directory) and a model file is in the Hugging Face cache layout, or sits next to a `config.json` whose `_name_or_path` looks like `org/name`, AIsbom asks `https://huggingface.co` for that repo's file list (`/api/models//tree/`, one request per page) and, if the file's hash is in that list, its model card (`/api/models/` or `/revision/`). Each repo is asked once per scan however many of its files are present. What is sent is the **repo id** and, for cached files, the **commit** from the cache path. The hash comparison happens on your machine, so no file paths, file names, hashes or findings are sent. A `_name_or_path` that is a single name, looks like a path, or names a directory that exists next to the file is never sent. A tree with none of this evidence makes no request. If `HF_TOKEN` is set it is sent to `huggingface.co` as for `hf://` scans, so matches in private repos you can access work too. This is a request to a third-party service, not telemetry, so `AISBOM_NO_TELEMETRY` does not affect it; `--no-hf-lookup`, `AISBOM_NO_HF_LOOKUP=1` or `--offline` does. Nothing is cached. +When a local scan writes CycloneDX 1.7 or SPDX 3.0 output (or `aisbom score` or `aisbom comply` scans a directory) and a model file is in the Hugging Face cache layout, or sits next to a `config.json` whose `_name_or_path` looks like `org/name`, AIsbom asks `https://huggingface.co` for that repo's file list (`/api/models//tree/`, one request per page) and, if the file's hash is in that list, its model card (`/api/models/` or `/revision/`). Each repo is asked once per scan however many of its files are present. What is sent is the **repo id** and, for cached files, the **commit** from the cache path. The hash comparison happens on your machine, so no file paths, file names, hashes or findings are sent. A `_name_or_path` that is a single name, looks like a path, or names a directory that exists next to the file is never sent. A tree with none of this evidence makes no request. If `HF_TOKEN` is set it is sent to `huggingface.co` as for `hf://` scans, so matches in private repos you can access work too. This is a request to a third-party service, not telemetry, so `AISBOM_NO_TELEMETRY` does not affect it; `--no-hf-lookup`, `AISBOM_NO_HF_LOOKUP=1` or `--offline` does. Nothing is cached. ### What's never collected @@ -734,7 +766,7 @@ export AISBOM_NO_TELEMETRY=1 AISBOM_NO_TELEMETRY=1 aisbom scan ./my-project ``` -`AISBOM_OFFLINE=1` (or `--offline` on `scan` and `score`) goes further: it disables telemetry and also every other network request — the update check and the PyPI, OSV and Hugging Face model-card lookups. +`AISBOM_OFFLINE=1` (or `--offline` on `scan`, `score` and `comply`) goes further: it disables telemetry and also every other network request — the update check and the PyPI, OSV and Hugging Face model-card lookups. ### Where the data goes diff --git a/aisbom/cli.py b/aisbom/cli.py index a491a88..ba574f8 100644 --- a/aisbom/cli.py +++ b/aisbom/cli.py @@ -16,6 +16,7 @@ from .diff import SBOMDiff from .cyclonedx_gen import build_cyclonedx_json from . import score as score_mod +from . import comply as comply_mod from .vex import ( baseline_findings, derive_statements, @@ -521,8 +522,9 @@ def _emit_vex( schema_version: str, dependencies: list[dict] | None = None, osv_enabled: bool = False, -) -> None: - """Write the VEX document(s) that accompany a just-written SBOM. +) -> list: + """Write the VEX document(s) that accompany a just-written SBOM, and + return the statements they carry. The serial number is read back out of the serialized SBOM rather than off the ``Bom`` object: it is the value the file on disk actually carries, and @@ -599,6 +601,8 @@ def _emit_vex( ) console.print(line + ".[/dim]") + return statements + def _generate_markdown(results: dict) -> str: """Render a GitHub-flavored Markdown report for CI artifacts.""" @@ -1345,6 +1349,57 @@ def bypass_scorecard( console.print("\n[green]Scorecard gate passed[/green] — no case below the floor.") +def _scan_whole_target( + target: str, + *, + strict: bool, + no_hf_lookup: bool, + progress: Console, + telemetry_threads: list, + refusal: str, + consequence: str, +) -> dict: + """Scan TARGET for a command that builds a document from the result, and + exit 1 unless every file was read. + + A fetch or parse failure is recorded in ``results["errors"]`` and the scan + returns normally. A document built from what survived would silently + describe a release missing the file that failed, so `score` and `comply` + refuse it, as `scan` already exits 1 on the same errors. + + The enrichment `scan` applies runs here too, so the document describes the + file a scan would have written. + """ + progress.print( + Panel.fit(f"🚀 [bold cyan]AIsbom[/bold cyan] Scanning: [underline]{target}[/underline]") + ) + try: + scanner = DeepScanner(target, strict_mode=strict, lint=False) + with progress.status("[cyan]Scanning...[/cyan]"): + results = scanner.scan() + except Exception as exc: + console.print(f"[bold red]✖ Scan failed:[/bold red] {exc}") + _flush_telemetry_threads(telemetry_threads) + raise typer.Exit(code=1) + + if results.get("errors"): + console.print( + f"[bold red]✖ {refusal}[/bold red] — " + f"{len(results['errors'])} target(s) could not be read:" + ) + for err in results["errors"]: + console.print( + f" [red]•[/red] {err.get('file', '?')}: {err.get('error', '?')}" + ) + console.print(f"\n[dim]{consequence}[/dim]") + _flush_telemetry_threads(telemetry_threads) + raise typer.Exit(code=1) + + _resolve_dependency_licenses(results, progress) + _match_local_model_cards(results, target, no_hf_lookup, progress) + return results + + def _render_score_footer() -> None: """Print the post-score footer, mirroring `scan`'s "Next steps" panel. @@ -1471,44 +1526,18 @@ def score( # machine-readable result *only*, so a progress panel on stdout would # make `aisbom score --json | jq` fail on the first line. progress = err_console if json_output else console - progress.print( - Panel.fit(f"🚀 [bold cyan]AIsbom[/bold cyan] Scanning: [underline]{target}[/underline]") + results = _scan_whole_target( + target, + strict=strict, + no_hf_lookup=no_hf_lookup, + progress=progress, + telemetry_threads=telemetry_threads, + refusal="Refusing to score a partial scan", + consequence=( + "A grade computed from only the artifacts that scanned " + "would overstate the document's completeness." + ), ) - try: - scanner = DeepScanner(target, strict_mode=strict, lint=False) - with progress.status("[cyan]Scanning...[/cyan]"): - results = scanner.scan() - except Exception as exc: - console.print(f"[bold red]✖ Scan failed:[/bold red] {exc}") - _flush_telemetry_threads(telemetry_threads) - raise typer.Exit(code=1) - - # A fetch or parse failure is recorded in results["errors"] and the - # scan returns normally, so the except above never fires. Grading what - # survived would hand back a completeness score for a document that is - # silently missing the shard that failed — precisely the claim this - # command exists to make trustworthy. `scan` already exits 1 on these; - # scoring must not be the softer path. - if results.get("errors"): - console.print( - "[bold red]✖ Refusing to score a partial scan[/bold red] — " - f"{len(results['errors'])} target(s) could not be read:" - ) - for err in results["errors"]: - console.print( - f" [red]•[/red] {err.get('file', '?')}: {err.get('error', '?')}" - ) - console.print( - "\n[dim]A grade computed from only the artifacts that scanned " - "would overstate the document's completeness.[/dim]" - ) - _flush_telemetry_threads(telemetry_threads) - raise typer.Exit(code=1) - - # The same enrichment `scan` applies, so the grade still describes the - # file a scan would have written (#129). - _resolve_dependency_licenses(results, progress) - _match_local_model_cards(results, target, no_hf_lookup, progress) doc = json.loads(build_cyclonedx_json(results, "1.7")) else: console.print( @@ -1606,6 +1635,147 @@ def score( _flush_telemetry_threads(telemetry_threads) +class ComplyFramework(str, Enum): + EU_AI_ACT = "eu-ai-act" + FDA = "fda" + + +@app.command() +def comply( + target: str = typer.Argument( + ..., help="What to scan: a directory, a model file, an hf:// slug, or an HTTP(S) URL." + ), + framework: ComplyFramework = typer.Option( + ..., + "--framework", + case_sensitive=False, + help=( + "eu-ai-act: Annex XI technical documentation + Article 53 training-content " + "summary. fda: §524B SBOM, VEX and component support table." + ), + ), + output_dir: str | None = typer.Option( + None, + "--output-dir", + help="Folder to write the pack into. Defaults to ./aisbom-comply-.", + ), + strict: bool = typer.Option( + False, help="Enable strict allowlisting mode (flags any unknown imports)." + ), + no_osv: bool = typer.Option( + False, + "--no-osv", + help=( + "For --framework fda, do not query OSV (api.osv.dev) for CVEs affecting " + "exact requirements.txt pins. AISBOM_NO_OSV=1 does the same." + ), + ), + no_hf_lookup: bool = typer.Option( + False, + "--no-hf-lookup", + help=( + "When TARGET is a directory, do not ask huggingface.co for the model " + "card of local model files matched to their repo. " + "AISBOM_NO_HF_LOOKUP=1 does the same." + ), + ), + offline_mode: bool = typer.Option( + False, + "--offline", + help=( + "Make no network access of any kind; remote targets are refused. " + "AISBOM_OFFLINE=1 does the same." + ), + ), +): + """ + Generate a regulator-facing documentation pack pre-filled from a scan. + + Fields are filled only from what the scan read, each tagged with its + source; everything else is left as a TODO for the provider. The pack + describes files, not your organisation, and is not legal advice. + """ + offline.enable(offline_mode) + if not offline.is_offline(): + t = threading.Thread(target=run_version_check_wrapper, daemon=True) + t.start() + _refuse_remote_target_offline(target) + + scan_id = uuid.uuid4().hex + telemetry_threads: list[threading.Thread | None] = [_maybe_emit_install_event()] + + if _classify_target(target) == "local" and not Path(target).exists(): + console.print( + f"[bold red]✖ Cannot build a pack from[/bold red] {target}: not an existing " + "file or directory, an hf:// slug or an HTTP(S) URL." + ) + _flush_telemetry_threads(telemetry_threads) + raise typer.Exit(code=1) + + results = _scan_whole_target( + target, + strict=strict, + no_hf_lookup=no_hf_lookup, + progress=console, + telemetry_threads=telemetry_threads, + refusal="Refusing to build a pack from a partial scan", + consequence=( + "Documentation built from only the files that scanned would " + "describe a release that is missing the ones that failed." + ), + ) + sbom_json = build_cyclonedx_json(results, "1.7") + doc = json.loads(sbom_json) + + out = Path(output_dir or f"aisbom-comply-{framework.value}") + out.mkdir(parents=True, exist_ok=True) + sbom_path = out / comply_mod.SBOM_FILE + sbom_path.write_text(sbom_json) + + vex_names: list[str] = [] + if framework == ComplyFramework.FDA: + vex_names = [Path(p).name for p in _vex_paths(str(sbom_path))] + statements = _emit_vex( + sbom_json=sbom_json, + artifacts=results["artifacts"], + dependencies=results.get("dependencies", []), + output=str(sbom_path), + vex_format=VexFormat.BOTH, + baseline_path=None, + schema_version="1.7", + osv_enabled=not (no_osv or osv.disabled_by_env() or offline.is_offline()), + ) + pack = comply_mod.build_fda_pack( + doc, + target=target, + vex_statements=statements, + vex_files=vex_names, + ) + title = "FDA §524B pack" + else: + pack = comply_mod.build_eu_ai_act_pack(doc, target=target) + title = "EU AI Act pack" + + for name, content in pack.items(): + (out / name).write_text(content) + + written = sorted([comply_mod.SBOM_FILE, *vex_names, *pack]) + console.print(f"\n[bold green]✔ {title} written to[/bold green] {out}/") + for name in written: + console.print(f" [dim]•[/dim] {name}") + console.print( + f"\n[dim]Start with {out / comply_mod.README_FILE}. Every " + f"'{comply_mod.TODO}' field needs your input, and pre-filled values " + "should be checked before use. This is not legal advice.[/dim]" + ) + + telemetry_threads.append(telemetry.post_event( + "cli_comply", {"framework": framework.value}, scan_id=scan_id, + )) + _check_update_status() + _flush_telemetry_threads(telemetry_threads) + + @app.command() def diff( old_file: str = typer.Argument(..., help="Path to baseline SBOM (JSON)"), diff --git a/aisbom/comply.py b/aisbom/comply.py new file mode 100644 index 0000000..68056b8 --- /dev/null +++ b/aisbom/comply.py @@ -0,0 +1,698 @@ +"""Regulator-facing documentation packs pre-filled from a scan (`aisbom comply`). + +Two frameworks: + +* ``eu-ai-act`` — an EU AI Act Annex XI technical-documentation skeleton and + an Article 53(1)(d) public summary of training content, laid out section by + section like the European Commission's template (published 24 July 2025). +* ``fda`` — the files an FD&C Act §524B premarket submission's SBOM package is + assembled from: the SBOM, VEX, and a component support table. + +The rule every builder here follows: **a field is filled only from something +the scan actually read, and says where it came from.** Everything else is a +``TODO (provider)`` placeholder. A regulator-facing document with a guessed +value in it is worse than one with a visible gap, so nothing is inferred — +not even values that look derivable, such as a parameter count from tensor +shapes, which a sharded or quantised release would get wrong. + +The builders read the serialized CycloneDX document rather than the scanner's +raw results. That is the file the pack bundles as ``sbom.json``, so every +pre-filled value can be traced back to a field a reader can inspect. + +Wording follows the aisbom.io compliance guides: the documents describe what +the files contain and never pass a verdict on the provider. The test suite +fails on the same verdict words the platform bans. +""" + +from __future__ import annotations + +import csv +import io +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +# Bump on every change to the wording or field mapping of a generated +# document, so a pack in someone's submission folder can be traced to the +# exact template that produced it. +TEMPLATE_VERSION = "2026-11-08" + +TODO = "TODO (provider)" + +SOURCE_HF = "Hugging Face model card" +SOURCE_HEADER = "model file header" +SOURCE_SCAN = "AIsbom scan" + +ANNEX_XI_FILE = "annex-xi-technical-documentation.md" +ART53_FILE = "art53-training-content-summary.md" +SUPPORT_FILE = "component-support.csv" +README_FILE = "README.md" +SBOM_FILE = "sbom.json" + +DISCLAIMER = ( + "AIsbom pre-filled this document from what a scan could read from the " + "model files. It does not assess, certify or attest to regulatory status, " + "and it describes files, not your processes, risk management or product " + "safety. Check every pre-filled value and complete every TODO before use. " + "This is not legal advice, and responsibility for these obligations rests " + "with you as the provider." +) + +_ML_COMPONENT_TYPE = "machine-learning-model" + +_EUR_LEX_AI_ACT = "https://eur-lex.europa.eu/eli/reg/2024/1689/oj" +_COMMISSION_TEMPLATE = ( + "https://digital-strategy.ec.europa.eu/en/library/" + "explanatory-notice-and-template-public-summary-training-content-general-purpose-ai-models" +) +_GUIDE_ART53 = "https://aisbom.io/compliance/eu-ai-act-article-53" +_GUIDE_FDA = "https://aisbom.io/compliance/fda-524b" + + +# --------------------------------------------------------------------------- +# Reading the SBOM +# --------------------------------------------------------------------------- + +def _components(doc: Dict[str, Any]) -> List[Dict[str, Any]]: + comps = doc.get("components") if isinstance(doc, dict) else None + if not isinstance(comps, list): + return [] + return [c for c in comps if isinstance(c, dict)] + + +def _props(entries: Any) -> List[Tuple[str, str]]: + if not isinstance(entries, list): + return [] + return [ + (str(p.get("name")), str(p.get("value"))) + for p in entries + if isinstance(p, dict) and p.get("name") and p.get("value") is not None + ] + + +def _prop_values(component: Dict[str, Any], name: str) -> List[str]: + return [v for n, v in _props(component.get("properties")) if n == name] + + +def _card(component: Dict[str, Any]) -> Dict[str, Any]: + card = component.get("modelCard") + return card if isinstance(card, dict) else {} + + +def _card_prop(component: Dict[str, Any], name: str) -> Optional[str]: + for n, v in _props(_card(component).get("properties")): + if n == name: + return v + return None + + +def _params(component: Dict[str, Any]) -> Dict[str, Any]: + params = _card(component).get("modelParameters") + return params if isinstance(params, dict) else {} + + +def _sha256(component: Dict[str, Any]) -> Optional[str]: + hashes = component.get("hashes") + if not isinstance(hashes, list): + return None + for entry in hashes: + if not isinstance(entry, dict): + continue + alg = str(entry.get("alg") or "").upper().replace("_", "-") + content = str(entry.get("content") or "") + if alg in ("SHA-256", "SHA256") and len(content) == 64: + return content + return None + + +def _licenses(component: Dict[str, Any]) -> List[str]: + found: List[str] = [] + entries = component.get("licenses") + if isinstance(entries, list): + for entry in entries: + if not isinstance(entry, dict): + continue + if entry.get("expression"): + found.append(str(entry["expression"])) + continue + lic = entry.get("license") + if isinstance(lic, dict): + value = lic.get("id") or lic.get("name") + if value: + found.append(str(value)) + return found + + +def _distinct(pairs: Iterable[Tuple[Optional[str], str]]) -> List[Tuple[str, str]]: + """Drop empty values and repeats, keeping first-seen order.""" + seen = set() + out: List[Tuple[str, str]] = [] + for value, source in pairs: + if not value or not str(value).strip(): + continue + key = (str(value).strip(), source) + if key in seen: + continue + seen.add(key) + out.append(key) + return out + + +@dataclass +class _Facts: + """Everything the documents may pre-fill, each value paired with its source.""" + + models: List[Dict[str, Any]] + dependencies: List[Dict[str, Any]] + repos: List[Tuple[str, str]] + versioned_names: List[Tuple[str, str]] + tasks: List[Tuple[str, str]] + families: List[Tuple[str, str]] + architectures: List[Tuple[str, str]] + datasets: List[Tuple[str, str]] + licenses: List[Tuple[str, str]] + libraries: List[Tuple[str, str]] + formats: List[Tuple[str, str]] + + +def _collect(doc: Dict[str, Any]) -> _Facts: + comps = _components(doc) + models = [c for c in comps if c.get("type") == _ML_COMPONENT_TYPE] + deps = [c for c in comps if c.get("type") != _ML_COMPONENT_TYPE] + + repos, names, tasks, families, archs = [], [], [], [], [] + datasets, licenses, libraries, formats = [], [], [], [] + for m in models: + repo = _card_prop(m, "aisbom:hf:repo_id") + # Card values come from Hugging Face whenever the card names a repo; + # without one, the only source the card builder has is the file header. + card_source = SOURCE_HF if repo else SOURCE_HEADER + params = _params(m) + if repo: + repos.append((repo, SOURCE_HF)) + revision = _card_prop(m, "aisbom:hf:revision") + names.append((f"{repo}@{revision}" if revision else repo, SOURCE_HF)) + tasks.append((params.get("task"), card_source)) + families.append((params.get("architectureFamily"), card_source)) + archs.append((params.get("modelArchitecture"), card_source)) + for ds in params.get("datasets") or []: + if isinstance(ds, dict): + datasets.append((ds.get("name"), card_source)) + for lic in _licenses(m): + licenses.append((lic, SOURCE_SCAN)) + licenses.append((_card_prop(m, "aisbom:hf:license"), SOURCE_HF)) + libraries.append((_card_prop(m, "aisbom:hf:library_name"), SOURCE_HF)) + for fmt in _prop_values(m, "aisbom:format"): + formats.append((fmt, SOURCE_SCAN)) + + return _Facts( + models=models, + dependencies=deps, + repos=_distinct(repos), + versioned_names=_distinct(names), + tasks=_distinct(tasks), + families=_distinct(families), + architectures=_distinct(archs), + datasets=_distinct(datasets), + licenses=_distinct(licenses), + libraries=_distinct(libraries), + formats=_distinct(formats), + ) + + +# --------------------------------------------------------------------------- +# Writing Markdown +# --------------------------------------------------------------------------- + +def _esc(value: Any) -> str: + """Make a value safe inside a Markdown table cell or inline span.""" + return ( + str(value) + .replace("\\", "\\\\") + .replace("|", "\\|") + .replace("\r", " ") + .replace("\n", " ") + ) + + +@dataclass +class _Writer: + lines: List[str] = field(default_factory=list) + filled: int = 0 + todo: int = 0 + + def heading(self, level: int, text: str) -> None: + if self.lines and self.lines[-1] != "": + self.lines.append("") + self.lines.extend([f"{'#' * level} {text}", ""]) + + def para(self, text: str) -> None: + if self.lines and self.lines[-1] != "": + self.lines.append("") + self.lines.extend([text, ""]) + + def fact(self, label: str, values: Sequence[Tuple[str, str]], hint: str = "") -> None: + """A field pre-filled from the scan, or a TODO when the scan had nothing.""" + if not values: + self.ask(label, hint) + return + self.filled += 1 + rendered = ", ".join(f"`{_esc(v)}` _({src})_" for v, src in values) + line = f"- **{label}:** {rendered}" + if hint: + line += f" — {hint}" + self.lines.append(line) + + def ask(self, label: str, hint: str = "") -> None: + """A field only the provider can answer.""" + self.todo += 1 + line = f"- **{label}:** **{TODO}**" + if hint: + line += f" — {hint}" + self.lines.append(line) + + def table(self, header: Sequence[str], rows: Sequence[Sequence[Any]]) -> None: + if self.lines and self.lines[-1] != "": + self.lines.append("") + self.lines.append("| " + " | ".join(header) + " |") + self.lines.append("|" + "|".join("---" for _ in header) + "|") + for row in rows: + self.lines.append("| " + " | ".join(_esc(c) for c in row) + " |") + self.lines.append("") + + def close(self) -> str: + while self.lines and self.lines[-1] == "": + self.lines.pop() + self.lines.extend(["", "---", "", DISCLAIMER]) + return "\n".join(self.lines).rstrip() + "\n" + + +def _intro(w: _Writer, title: str, target: str, *, legend: bool = True) -> None: + w.lines.extend([f"# {title}", ""]) + w.para( + f"Generated by AIsbom from a scan of `{_esc(target)}`. " + f"Template version {TEMPLATE_VERSION}." + ) + if legend: + w.para( + f"Values tagged with a source in italics were read by the scan. Fields " + f"marked **{TODO}** need information a file scan cannot read; " + f"AIsbom leaves them blank rather than guess." + ) + + +# --------------------------------------------------------------------------- +# EU AI Act — Annex XI +# --------------------------------------------------------------------------- + +# Every per-format threat property the scanner writes (see `properties.py`), +# with the label the summary gives it. +_THREAT_PROPERTIES = ( + ("aisbom:pickle:opcode", "dangerous imports"), + ("aisbom:keras:threat", "Keras threats"), + ("aisbom:gguf:chat_template_threat", "chat template threats"), + ("aisbom:onnx:threat", "ONNX threats"), +) + +_ELEVATED_RISKS = ("CRITICAL", "HIGH") + + +def _findings(component: Dict[str, Any]) -> str: + notes: List[str] = [] + for prop, label in _THREAT_PROPERTIES: + values = _prop_values(component, prop) + if values: + notes.append(f"{label}: " + ", ".join(values)) + if _prop_values(component, "aisbom:unreadable"): + notes.append("not readable by any parser; not inspected") + if _prop_values(component, "aisbom:pickle:scan_incomplete"): + notes.append("pickle scan incomplete") + if notes: + return "; ".join(notes) + # A threat property this module does not know yet must not read as a + # clean bill of health beside an elevated rating. + if any(r.upper().startswith(_ELEVATED_RISKS) for r in _prop_values(component, "aisbom:risk")): + return f"see the AIsbom risk rating and {SBOM_FILE}" + return "none" + + +def _annex_xi(facts: _Facts, target: str) -> _Writer: + w = _Writer() + _intro(w, "Technical documentation — EU AI Act Annex XI (skeleton)", target) + w.para( + "Article 53(1)(a) of Regulation (EU) 2024/1689 asks providers of " + "general-purpose AI models to draw up technical documentation containing " + "at least the information in Annex XI, and to keep it up to date. " + "Section 1 applies to every such provider; Section 2 only to models " + "with systemic risk." + ) + + w.heading(2, "Section 1 — Information for all providers of general-purpose AI models") + w.para("**Point 1 — General description of the model.**") + + w.heading(3, "Point 1(a) — Intended tasks and the AI systems the model can be integrated into") + w.fact("Task declared in the model card", facts.tasks, + "confirm it matches the tasks you intend the model to perform") + w.ask("Intended tasks", "what the model is intended to do") + w.ask("Type and nature of AI systems it can be integrated into") + + w.heading(3, "Point 1(b) — Acceptable use policies") + w.ask("Applicable acceptable use policies") + + w.heading(3, "Point 1(c) — Date of release and methods of distribution") + w.ask("Date of release") + w.fact("Hugging Face repository the scanned files come from", facts.repos) + w.ask("Methods of distribution") + + w.heading(3, "Point 1(d) — Architecture and number of parameters") + w.fact("Architecture family", facts.families) + w.fact("Model architecture", facts.architectures) + w.ask("Number of parameters", "AIsbom does not derive this from tensor shapes") + + w.heading(3, "Point 1(e) — Modality and format of inputs and outputs") + w.ask("Input modality and format") + w.ask("Output modality and format") + + w.heading(3, "Point 1(f) — Licence") + w.fact("Licence", facts.licenses) + + w.para("**Point 2 — Detailed description of the elements in point 1 and of the development process.**") + + w.heading(3, "Point 2(a) — Technical means needed for integration") + w.fact("Library declared in the model card", facts.libraries) + w.fact("Model file formats in this release", facts.formats) + w.ask("Instructions of use, infrastructure and tools needed to integrate the model") + + w.heading(3, "Point 2(b) — Design specifications and training process") + w.ask("Training methodologies and techniques") + w.ask("Key design choices, with rationale and assumptions") + w.ask("What the model is designed to optimise for, and the relevance of its parameters") + + w.heading(3, "Point 2(c) — Data used for training, testing and validation") + w.fact("Training datasets declared in the model card", facts.datasets, + "confirm each one, and list any the card omits") + w.ask("Type and provenance of the data, and curation methodologies") + w.ask("Number of data points, scope and main characteristics") + w.ask("How the data was obtained and selected") + w.ask("Measures to detect unsuitable data sources and identifiable biases") + + w.heading(3, "Point 2(d) — Computational resources used to train the model") + w.ask("Computational resources", "for example the number of floating point operations") + w.ask("Training time and other relevant training details") + + w.heading(3, "Point 2(e) — Energy consumption") + w.ask("Known or estimated energy consumption", + "may be estimated from the computational resources in point 2(d)") + + w.heading(2, "Section 2 — Additional information for models with systemic risk") + w.para( + "Complete this section only for a general-purpose AI model with systemic " + "risk (Article 51); otherwise mark it not applicable." + ) + w.heading(3, "Item 1 — Evaluation strategies and results") + w.ask("Evaluation strategies, criteria, metrics and results") + w.heading(3, "Item 2 — Adversarial testing and model adaptations") + w.ask("Adversarial testing measures (for example red teaming), alignment and fine-tuning") + w.heading(3, "Item 3 — System architecture") + w.ask("How the software components build on or feed into each other", + "where applicable") + + w.heading(2, "Appendix A — Scanned model files") + note = ( + "AIsbom's mapping: the Act does not name hashes, but a hash pins exactly " + "which file this documentation describes." + ) + if any(_sha256(m) is None for m in facts.models): + note += ( + " A remote scan reads files without downloading them whole, so it " + "computes no hash; scan a local copy to record one." + ) + w.para(note) + if facts.models: + w.table( + ["File", "Format", "SHA-256", "AIsbom risk", "Security findings"], + [ + [ + m.get("name") or "", + ", ".join(_prop_values(m, "aisbom:format")) or "unknown", + _sha256(m) or "not computed", + ", ".join(_prop_values(m, "aisbom:risk")) or "unknown", + _findings(m), + ] + for m in facts.models + ], + ) + else: + w.para("The scan found no model files.") + + w.heading(2, "Appendix B — Software dependencies") + w.para( + "Article 53(1)(b) and Annex XII expect integrators to receive the " + "versions of software relevant to using the model. These are the " + "dependencies the scan found." + ) + if facts.dependencies: + w.table( + ["Package", "Version", "Licence"], + [ + [d.get("name") or "", d.get("version") or "unpinned", + ", ".join(_licenses(d)) or "unknown"] + for d in facts.dependencies + ], + ) + else: + w.para("The scan found no requirements.txt dependencies.") + return w + + +# --------------------------------------------------------------------------- +# EU AI Act — Article 53(1)(d) public summary of training content +# --------------------------------------------------------------------------- + +_YES_NO = "Yes / No" +_MODALITY = "Text / Image / Audio / Video / Other" + + +def _art53(facts: _Facts, target: str) -> _Writer: + w = _Writer() + _intro(w, "Public summary of training content — EU AI Act Article 53(1)(d) (draft)", target) + w.para( + "Follows the sections of the European Commission's *Template for the " + "Public Summary of Training Content for General-Purpose AI models* " + f"(published 24 July 2025, {_COMMISSION_TEMPLATE}). The Commission's " + "template is the form providers use; this draft follows its numbering " + "so answers can be transferred section by section." + ) + w.ask("Version of the summary", "with links to previous versions where applicable") + w.ask("Last update") + + w.heading(2, "1 General information") + w.heading(3, "1.1 Provider identification") + w.ask("Provider name and contact details") + w.ask("Authorised representative name and contact details", + "only if the provider is established outside the Union (Article 54)") + + w.heading(3, "1.2 Model identification") + w.fact("Versioned model name(s)", facts.versioned_names, + "replace with the versioned name you place on the market if it differs") + w.fact("Model card link", + [(f"https://huggingface.co/{repo}", src) for repo, src in facts.repos]) + w.ask("Model dependencies", + "if the model modifies or fine-tunes another general-purpose AI model, " + "name it and link its summary") + w.ask("Date of placement of the model on the Union market") + + w.heading(3, "1.3 Modalities, overall training data size and other characteristics") + w.ask("Modalities present in the training data", _MODALITY) + w.ask("Training data size per modality", "the template's size ranges, or another unit") + w.ask("Types of content per modality") + w.ask("Latest date of data acquisition or collection", "MM/YYYY, and whether training continues on new data") + w.ask("Linguistic characteristics of the overall training data") + w.ask("Other relevant characteristics of the overall training data") + + w.heading(2, "2 List of data sources") + w.heading(3, "2.1 Publicly available datasets") + w.ask("Have you used publicly available datasets to train the model?", _YES_NO) + w.fact("Datasets declared in the model card", facts.datasets, + "confirm each was used, add an access link, and check whether it is " + "'large' (over 3% of the public-dataset data for a modality)") + w.ask("List of large publicly available datasets", "name and access link for each") + w.ask("General description of other publicly available datasets") + + w.heading(3, "2.2 Private non-publicly available datasets obtained from third parties") + w.heading(4, "2.2.1 Datasets commercially licensed by rightsholders or their representatives") + w.ask("Transactional commercial licensing agreements concluded?", _YES_NO) + w.ask("Modalities covered", _MODALITY) + w.heading(4, "2.2.2 Private datasets obtained from other third parties") + w.ask("Private datasets obtained from other third parties?", _YES_NO) + w.ask("Modalities covered", _MODALITY) + w.ask("List of those datasets, if publicly known") + w.ask("General description of those not publicly known") + + w.heading(3, "2.3 Data crawled and scraped from online sources") + w.ask("Were crawlers used by the provider or on its behalf?", _YES_NO) + w.ask("Crawler name(s) or identifier(s)") + w.ask("Purposes of the crawler(s)") + w.ask("General description of crawler behaviour", "for example robots.txt, paywalls, captchas") + w.ask("Period of data collection", "from MM/YYYY to MM/YYYY") + w.ask("Description of the type of content and online sources crawled") + w.ask("Modalities covered", _MODALITY) + w.ask("Most relevant domain names crawled", + "top 10% by content size; SMEs top 5% or 1,000, whichever is lower") + + w.heading(3, "2.4 User data") + w.ask("Was data from user interactions with the model used to train it?", _YES_NO) + w.ask("Was data from user interactions with your other services or products used?", _YES_NO) + w.ask("Description of the services or products the user data came from") + w.ask("Modalities covered", _MODALITY) + + w.heading(3, "2.5 Synthetic data") + w.ask("Was synthetic AI-generated data created by you or on your behalf to train the model?", _YES_NO) + w.ask("Modalities of the synthetic data", _MODALITY) + w.ask("General-purpose AI model(s) on the market used to generate it") + w.ask("Other AI models used to generate it, including your own") + + w.heading(3, "2.6 Other sources of data") + w.ask("Were data sources other than those in sections 2.1 to 2.5 used?", _YES_NO) + w.ask("Narrative description of those sources and data") + + w.heading(2, "3 Data processing aspects") + w.heading(3, "3.1 Respect of reservation of rights from the text and data mining exception or limitation") + w.ask("Signatory to the Code of Practice for general-purpose AI models?", _YES_NO) + w.ask("Measures taken to respect reservations of rights before and during data collection") + + w.heading(3, "3.2 Removal of illegal content") + w.ask("General description of measures taken") + + w.heading(3, "3.3 Other information (optional)") + w.ask("Other relevant information about data processing", "optional") + return w + + +def _eu_readme(target: str, annex: _Writer, art53: _Writer) -> str: + w = _Writer() + _intro(w, "EU AI Act documentation pack", target) + w.table( + ["File", "What it is"], + [ + [ANNEX_XI_FILE, "Annex XI technical documentation skeleton (Article 53(1)(a))"], + [ART53_FILE, "Public summary of training content, in the Commission template's sections (Article 53(1)(d))"], + [SBOM_FILE, "The CycloneDX 1.7 SBOM every pre-filled value was read from"], + ], + ) + filled = annex.filled + art53.filled + todo = annex.todo + art53.todo + w.para(f"{filled} field(s) pre-filled from the scan, {todo} left as TODO for the provider.") + w.para( + "A file scan can read what the model files and their Hugging Face model " + "card declare: identifiers, task, architecture, licence, declared " + "datasets, file formats, hashes and security findings. It cannot read " + "the provider's identity, release plans, training process, compute, " + "energy use, data collection or copyright measures." + ) + w.heading(2, "Sources") + w.lines.extend([ + f"- Regulation (EU) 2024/1689: {_EUR_LEX_AI_ACT}", + f"- Commission template for the public summary of training content: {_COMMISSION_TEMPLATE}", + f"- AIsbom guide to Article 53: {_GUIDE_ART53}", + ]) + return w.close() + + +def build_eu_ai_act_pack(doc: Dict[str, Any], *, target: str) -> Dict[str, str]: + """Return ``{filename: content}`` for the EU AI Act pack (SBOM excluded).""" + facts = _collect(doc) + annex = _annex_xi(facts, target) + art53 = _art53(facts, target) + readme = _eu_readme(target, annex, art53) + return { + ANNEX_XI_FILE: annex.close(), + ART53_FILE: art53.close(), + README_FILE: readme, + } + + +# --------------------------------------------------------------------------- +# FDA §524B +# --------------------------------------------------------------------------- + +_SUPPORT_COLUMNS = [ + "bom_ref", "name", "type", "version", "sha256", "license", + "support_level", "end_of_support_date", +] + + +def _support_csv(doc: Dict[str, Any]) -> str: + buf = io.StringIO() + writer = csv.DictWriter(buf, fieldnames=_SUPPORT_COLUMNS, lineterminator="\n") + writer.writeheader() + for c in _components(doc): + writer.writerow({ + "bom_ref": c.get("bom-ref") or "", + "name": c.get("name") or "", + "type": c.get("type") or "", + "version": c.get("version") or "", + "sha256": _sha256(c) or "", + "license": ";".join(_licenses(c)), + "support_level": TODO, + "end_of_support_date": TODO, + }) + return buf.getvalue() + + +def build_fda_pack( + doc: Dict[str, Any], + *, + target: str, + vex_statements: Sequence[Any], + vex_files: Sequence[str], +) -> Dict[str, str]: + """Return ``{filename: content}`` for the FDA pack (SBOM and VEX excluded, + since the CLI writes those with the same code `scan --vex` uses).""" + comps = _components(doc) + statuses = [getattr(s, "status", None) for s in vex_statements] + affected = statuses.count("affected") + + w = _Writer() + _intro(w, "FDA §524B documentation pack", target, legend=False) + w.para( + "A premarket submission for a cyber device must include an SBOM " + "(FD&C Act §524B(b)(3)). FDA's premarket cybersecurity guidance " + "(2026-02-03, nonbinding) recommends a machine-readable SBOM with the " + "NTIA minimum elements, the support status of each component, and an " + "assessment of each known vulnerability. This pack holds the parts a " + "scan of the model files can produce." + ) + w.table( + ["File", "What it is"], + [ + [SBOM_FILE, f"CycloneDX 1.7 SBOM, {len(comps)} component(s)"], + *[[name, "VEX for the SBOM's components"] for name in vex_files], + [SUPPORT_FILE, "One row per component, with support level and end-of-support date to fill in"], + ], + ) + w.heading(2, "Vulnerability assessment") + w.para( + "FDA's guidance asks for a risk assessment of each known vulnerability " + "in the device's components; a VEX is one machine-readable way to " + f"record it. The VEX files carry {len(statuses)} statement(s), " + f"{affected} affected. Model-file findings are keyed on AIsbom finding " + "classes; CVE statements cover exact `==` pins in requirements.txt, " + "looked up in OSV unless the run was offline or used --no-osv." + ) + w.heading(2, "What this pack cannot evidence") + w.lines.extend([ + f"- **Support level and end-of-support date** for each component: " + f"`{SUPPORT_FILE}` marks both {TODO}.", + "- **Supplier name** (an NTIA minimum element): the scan does not record " + "a supplier for model files.", + "- **Software outside the scanned target**, such as firmware, the " + "operating system or application code.", + "- **Vulnerabilities without a CVE feed entry**, and CVEs in unpinned " + "dependencies.", + ]) + w.heading(2, "Sources") + w.lines.append(f"- AIsbom guide to FDA §524B: {_GUIDE_FDA}") + return { + SUPPORT_FILE: _support_csv(doc), + README_FILE: w.close(), + } diff --git a/docs/comply.md b/docs/comply.md new file mode 100644 index 0000000..ce8e3a6 --- /dev/null +++ b/docs/comply.md @@ -0,0 +1,88 @@ +# `aisbom comply` field mapping + +Every field `aisbom comply` writes, and where its value comes from. A field +with no scan source is always written as `TODO (provider)`. A field with a +scan source falls back to `TODO (provider)` when the scan found nothing for +it. Template version: `2026-11-08` (`aisbom/comply.py`, `TEMPLATE_VERSION`). + +Sources, as tagged in the generated documents: + +- **Hugging Face model card**: the `modelCard` block of a model component in + `sbom.json`. It is present for `hf://` scans and for local files whose hash + matched a file in a Hugging Face repo. +- **model file header**: the same `modelCard` fields when no repo was matched. + Today only a GGUF header supplies one (its architecture). +- **AIsbom scan**: properties the scanner wrote from the file itself, such as + `aisbom:format`, or a licence it read from file metadata. + +Wording follows the [aisbom.io compliance guides](https://aisbom.io/compliance/). +Lines marked *AIsbom's mapping* are interpretations, not quotations of the +text. + +## EU AI Act: Annex XI technical documentation + +Regulation (EU) 2024/1689, Article 53(1)(a) and Annex XI. + +| Annex XI | Field | Value from | +|---|---|---| +| 1(a) | Task declared in the model card | `modelParameters.task` | +| 1(a) | Intended tasks | TODO | +| 1(a) | Type and nature of AI systems it can be integrated into | TODO | +| 1(b) | Applicable acceptable use policies | TODO | +| 1(c) | Date of release | TODO | +| 1(c) | Hugging Face repository the scanned files come from | `aisbom:hf:repo_id` | +| 1(c) | Methods of distribution | TODO | +| 1(d) | Architecture family | `modelParameters.architectureFamily` | +| 1(d) | Model architecture | `modelParameters.modelArchitecture` | +| 1(d) | Number of parameters | TODO (never derived from tensor shapes) | +| 1(e) | Input / output modality and format | TODO | +| 1(f) | Licence | component `licenses[]`, then `aisbom:hf:license` | +| 2(a) | Library declared in the model card | `aisbom:hf:library_name` | +| 2(a) | Model file formats in this release (*AIsbom's mapping*) | `aisbom:format` | +| 2(a) | Instructions of use, infrastructure and tools | TODO | +| 2(b) | Training methodologies, design choices, optimisation target | TODO | +| 2(c) | Training datasets declared in the model card | `modelParameters.datasets[].name` | +| 2(c) | Provenance and curation, data points, selection, bias measures | TODO | +| 2(d) | Computational resources, training time | TODO | +| 2(e) | Known or estimated energy consumption | TODO | +| Section 2, items 1–3 | Evaluation, adversarial testing, system architecture (systemic-risk models only) | TODO | +| Appendix A (*AIsbom's mapping*) | Scanned files: format, SHA-256, AIsbom risk, security findings | component `hashes`, `aisbom:format`, `aisbom:risk`, `aisbom:pickle:opcode`, `aisbom:unreadable` | +| Appendix B | Software dependencies: package, version, licence (Article 53(1)(b), Annex XII) | `library` components | + +## EU AI Act: public summary of training content + +Article 53(1)(d), following the sections of the Commission's *Template for the +Public Summary of Training Content for General-Purpose AI models* (24 July +2025). Every field the template asks for is present, in its order. + +| Template section | Pre-filled field | Value from | +|---|---|---| +| 1.2 | Versioned model name(s) | `aisbom:hf:repo_id` `@` `aisbom:hf:revision` | +| 1.2 | Model card link | `https://huggingface.co/` + `aisbom:hf:repo_id` | +| 2.1 | Datasets declared in the model card, for the provider to confirm | `modelParameters.datasets[].name` | + +Everything else, including the version and date of the summary, all of 1.1, +model dependencies and market placement date (1.2), all of 1.3, every Yes/No +question, and sections 2.2 to 3.3, is TODO. + +## FDA §524B + +FD&C Act §524B(b)(3) and FDA's premarket cybersecurity guidance (2026-02-03, +nonbinding). The guidance does not name VEX; the pack offers VEX as one +machine-readable way to record the per-vulnerability assessment it asks for. + +| File | Content | +|---|---| +| `sbom.json` | The CycloneDX 1.7 SBOM `aisbom scan` writes for the same target | +| `sbom.openvex.json`, `sbom.vex.cdx.json` | The VEX `aisbom scan --vex` writes: AIsbom finding classes for model files, plus OSV CVE statements for exact `==` pins unless `--no-osv` or `--offline` | +| `component-support.csv` | One row per component: `bom_ref`, `name`, `type`, `version`, `sha256`, `license` from the SBOM; `support_level` and `end_of_support_date` always TODO | +| `README.md` | Contents, VEX statement counts, and what the pack cannot evidence: support and end-of-support data, supplier name, software outside the target, vulnerabilities with no CVE entry | + +## Guards + +- `tests/test_comply.py` fails if any generated file contains verdict + language such as "compliant", "certified" or "guarantee", the same list the + aisbom.io guides are held to. +- Every generated Markdown file states its template version and ends with the + same "not legal advice" disclaimer. +- A scan in which any target could not be read writes no pack and exits `1`. diff --git a/tests/test_comply.py b/tests/test_comply.py new file mode 100644 index 0000000..34b8dc3 --- /dev/null +++ b/tests/test_comply.py @@ -0,0 +1,504 @@ +"""Tests for `aisbom comply` — regulator-facing documentation packs.""" + +import csv +import io +import json +import re + +import pytest +from typer.testing import CliRunner + +from aisbom import comply +from aisbom.cli import app + +runner = CliRunner() + + +# The verdict words the aisbom.io compliance guides are held to. The generated +# documents describe files, so wording that reads as a regulatory verdict +# about the provider is banned here too. Keep the two lists in step. +VERDICT_WORDS = [ + "compliant", + "compliance verdict", + "certified", + "certifies", + "guarantee", + "guaranteed", + "meets the requirement", + "satisfies the requirement", + "legally", + "approved by", + "audit-proof", +] + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +def _hf_sbom(): + """The shape an `hf://` scan writes: model cards, no hashes.""" + card = { + "modelParameters": { + "task": "fill-mask", + "architectureFamily": "bert", + "modelArchitecture": "BertForMaskedLM", + "datasets": [ + {"type": "dataset", "name": "bookcorpus"}, + {"type": "dataset", "name": "wikipedia"}, + ], + }, + "properties": [ + {"name": "aisbom:hf:repo_id", "value": "google-bert/bert-base-uncased"}, + {"name": "aisbom:hf:license", "value": "apache-2.0"}, + {"name": "aisbom:hf:library_name", "value": "transformers"}, + {"name": "aisbom:hf:revision", "value": "86b5e09"}, + ], + } + return { + "bomFormat": "CycloneDX", + "specVersion": "1.7", + "serialNumber": "urn:uuid:33333333-3333-3333-3333-333333333333", + "metadata": {"timestamp": "2026-11-08T00:00:00+00:00"}, + "components": [ + { + "type": "machine-learning-model", + "name": "model.safetensors", + "bom-ref": "artifact-0-model.safetensors", + "properties": [ + {"name": "aisbom:format", "value": "safetensors"}, + {"name": "aisbom:risk", "value": "LOW"}, + ], + "modelCard": card, + }, + { + "type": "machine-learning-model", + "name": "model.onnx", + "bom-ref": "artifact-1-model.onnx", + "properties": [ + {"name": "aisbom:format", "value": "onnx"}, + {"name": "aisbom:risk", "value": "LOW"}, + ], + "modelCard": card, + }, + ], + } + + +def _local_sbom(): + """The shape a local scan with no Hugging Face match writes: hashes, a + dangerous pickle, a pinned dependency, and no model card at all.""" + return { + "bomFormat": "CycloneDX", + "specVersion": "1.7", + "serialNumber": "urn:uuid:44444444-4444-4444-4444-444444444444", + "metadata": {"timestamp": "2026-11-08T00:00:00+00:00"}, + "components": [ + { + "type": "machine-learning-model", + "name": "weights|v2.pt", + "bom-ref": "artifact-0-weights|v2.pt", + "hashes": [{"alg": "SHA-256", "content": "a" * 64}], + "properties": [ + {"name": "aisbom:format", "value": "pickle"}, + {"name": "aisbom:risk", "value": "CRITICAL"}, + {"name": "aisbom:pickle:opcode", "value": "posix.system"}, + ], + }, + { + "type": "library", + "name": "torch", + "version": "2.10.0", + "bom-ref": "dependency-0-torch", + "licenses": [{"license": {"id": "BSD-3-Clause"}}], + "purl": "pkg:pypi/torch@2.10.0", + }, + ], + } + + +def _all_text(pack): + return "\n".join(pack.values()) + + +def _eu(doc): + return comply.build_eu_ai_act_pack(doc, target="hf://google-bert/bert-base-uncased") + + +def _fda(doc, statements=()): + return comply.build_fda_pack( + doc, + target="./models", + vex_statements=list(statements), + vex_files=["sbom.openvex.json", "sbom.vex.cdx.json"], + ) + + +# --------------------------------------------------------------------------- +# EU AI Act pack: structure +# --------------------------------------------------------------------------- + +def test_eu_pack_writes_the_two_documents_and_an_index(): + assert set(_eu(_hf_sbom())) == { + comply.ANNEX_XI_FILE, + comply.ART53_FILE, + comply.README_FILE, + } + + +ART53_SECTIONS = [ + "1.1", "1.2", "1.3", + "2.1", "2.2", "2.2.1", "2.2.2", "2.3", "2.4", "2.5", "2.6", + "3.1", "3.2", "3.3", +] + + +def test_art53_follows_the_commission_template_section_order(): + text = _eu(_hf_sbom())[comply.ART53_FILE] + headings = re.findall(r"^#{2,4} (\d(?:\.\d)+)\.? ", text, flags=re.M) + assert headings == ART53_SECTIONS + + +def test_art53_names_the_template_it_follows(): + text = _eu(_hf_sbom())[comply.ART53_FILE] + assert "Template for the Public Summary of Training Content" in text + assert "24 July 2025" in text + + +ANNEX_XI_POINTS = [ + "1(a)", "1(b)", "1(c)", "1(d)", "1(e)", "1(f)", + "2(a)", "2(b)", "2(c)", "2(d)", "2(e)", +] + + +def test_annex_xi_covers_every_section_1_point_in_order(): + text = _eu(_hf_sbom())[comply.ANNEX_XI_FILE] + found = re.findall(r"^### Point (\d\([a-f]\))", text, flags=re.M) + assert found == ANNEX_XI_POINTS + + +def test_annex_xi_carries_the_systemic_risk_section(): + text = _eu(_hf_sbom())[comply.ANNEX_XI_FILE] + assert "Section 2" in text + assert "systemic risk" in text + for item in ("Evaluation strategies", "Adversarial testing", "System architecture"): + assert item in text + + +# --------------------------------------------------------------------------- +# EU AI Act pack: what is pre-filled, and only from what the scan read +# --------------------------------------------------------------------------- + +def test_model_card_facts_are_prefilled_with_their_source(): + pack = _eu(_hf_sbom()) + annex = pack[comply.ANNEX_XI_FILE] + art53 = pack[comply.ART53_FILE] + + assert "fill-mask" in annex + assert "BertForMaskedLM" in annex + assert "apache-2.0" in annex + assert "transformers" in annex + assert "bookcorpus" in annex and "wikipedia" in annex + assert "google-bert/bert-base-uncased@86b5e09" in art53 + assert "bookcorpus" in art53 + assert comply.SOURCE_HF in annex + + +def test_identical_values_across_artifacts_are_listed_once(): + annex = _eu(_hf_sbom())[comply.ANNEX_XI_FILE] + point = _section(annex, "### Point 1(a)", "### Point 1(b)") + assert point.count("fill-mask") == 1 + + +def test_a_card_free_scan_leaves_model_facts_as_todo(): + pack = _eu(_local_sbom()) + annex = pack[comply.ANNEX_XI_FILE] + for point, nxt in (("1(a)", "1(b)"), ("1(d)", "1(e)"), ("1(f)", "2(a)")): + body = _section(annex, f"### Point {point}", f"### Point {nxt}") + assert comply.TODO in body + assert comply.SOURCE_HF not in body + art53 = pack[comply.ART53_FILE] + model_id = _section(art53, "### 1.2", "### 1.3") + assert comply.TODO in model_id + + +def test_parameter_count_is_never_prefilled(): + """The scan cannot read it reliably, so it is always left to the provider.""" + annex = _eu(_hf_sbom())[comply.ANNEX_XI_FILE] + body = _section(annex, "### Point 1(d)", "### Point 1(e)") + params_line = next(line for line in body.splitlines() if "Number of parameters" in line) + assert comply.TODO in params_line + + +def test_provider_identity_is_always_todo(): + art53 = _eu(_hf_sbom())[comply.ART53_FILE] + body = _section(art53, "### 1.1", "### 1.2") + assert body.count(comply.TODO) >= 2 + + +def test_declared_datasets_are_offered_for_confirmation_not_asserted(): + art53 = _eu(_hf_sbom())[comply.ART53_FILE] + body = _section(art53, "### 2.1", "### 2.2 ") + assert "bookcorpus" in body + assert "confirm" in body.lower() + # Whether public datasets were used is the provider's answer to give. + used_line = next(line for line in body.splitlines() + if "publicly available datasets to train" in line) + assert comply.TODO in used_line + + +def test_artifact_inventory_shows_hashes_and_marks_unhashed_files(): + local = _eu(_local_sbom())[comply.ANNEX_XI_FILE] + assert "a" * 64 in local + remote = _eu(_hf_sbom())[comply.ANNEX_XI_FILE] + assert "not computed" in remote + + +def test_security_findings_are_listed_with_their_opcodes(): + annex = _eu(_local_sbom())[comply.ANNEX_XI_FILE] + assert "CRITICAL" in annex + assert "posix.system" in annex + + +def _model_with(fmt, risk, props): + return { + "components": [{ + "type": "machine-learning-model", + "name": f"m.{fmt}", + "bom-ref": f"artifact-0-m.{fmt}", + "properties": [ + {"name": "aisbom:format", "value": fmt}, + {"name": "aisbom:risk", "value": risk}, + *[{"name": n, "value": v} for n, v in props], + ], + }], + } + + +@pytest.mark.parametrize("fmt,prop,value", [ + ("keras", "aisbom:keras:threat", "Lambda layer: marshalled bytecode"), + ("gguf", "aisbom:gguf:chat_template_threat", "jinja: __globals__ access"), + ("onnx", "aisbom:onnx:threat", "custom operator domain: evil.ops"), +]) +def test_non_pickle_threats_are_listed(fmt, prop, value): + """The summary must never contradict the SBOM it bundles.""" + annex = _eu(_model_with(fmt, "CRITICAL", [(prop, value)]))[comply.ANNEX_XI_FILE] + row = next(line for line in annex.splitlines() if line.startswith(f"| m.{fmt}")) + assert value in row + assert "| none |" not in row + + +def test_a_high_risk_file_never_reads_none(): + """A threat property this module does not know yet still cannot surface + as 'none' beside a CRITICAL rating.""" + annex = _eu(_model_with("newfmt", "CRITICAL (something)", []))[comply.ANNEX_XI_FILE] + row = next(line for line in annex.splitlines() if line.startswith("| m.newfmt")) + assert "| none |" not in row + assert "sbom.json" in row + + +def test_pipes_in_file_names_do_not_break_tables(): + annex = _eu(_local_sbom())[comply.ANNEX_XI_FILE] + row = next(line for line in annex.splitlines() if "weights" in line and line.startswith("|")) + assert "weights\\|v2.pt" in row + + +def test_software_dependencies_are_listed_for_integrators(): + annex = _eu(_local_sbom())[comply.ANNEX_XI_FILE] + assert "torch" in annex and "2.10.0" in annex and "BSD-3-Clause" in annex + + +def test_eu_index_counts_prefilled_and_todo_fields(): + readme = _eu(_hf_sbom())[comply.README_FILE] + assert re.search(r"\d+ field\(s\) pre-filled", readme) + assert re.search(r"\d+ left as TODO", readme) + assert comply.ANNEX_XI_FILE in readme and comply.ART53_FILE in readme + + +# --------------------------------------------------------------------------- +# FDA pack +# --------------------------------------------------------------------------- + +def test_fda_pack_writes_support_table_and_index(): + assert set(_fda(_local_sbom())) == {comply.SUPPORT_FILE, comply.README_FILE} + + +def test_support_table_has_a_row_per_component_with_todo_lifecycle(): + table = _fda(_local_sbom())[comply.SUPPORT_FILE] + rows = list(csv.DictReader(io.StringIO(table))) + assert [r["name"] for r in rows] == ["weights|v2.pt", "torch"] + for row in rows: + assert row["support_level"] == comply.TODO + assert row["end_of_support_date"] == comply.TODO + torch = rows[1] + assert torch["version"] == "2.10.0" + assert torch["license"] == "BSD-3-Clause" + assert rows[0]["sha256"] == "a" * 64 + + +def test_fda_index_lists_bundle_and_vex_counts(): + class _S: + def __init__(self, status): + self.status = status + + readme = _fda(_local_sbom(), [_S("affected"), _S("not_affected"), _S("not_affected")])[ + comply.README_FILE + ] + for name in ("sbom.json", "sbom.openvex.json", "sbom.vex.cdx.json", comply.SUPPORT_FILE): + assert name in readme + assert "3 statement(s)" in readme + assert "1 affected" in readme + + +def test_fda_index_does_not_claim_vex_is_required(): + readme = _fda(_local_sbom())[comply.README_FILE].lower() + assert "vex required" not in readme + assert "requires vex" not in readme + assert "one machine-readable way" in readme + + +def test_fda_index_names_what_it_cannot_evidence(): + readme = _fda(_local_sbom())[comply.README_FILE] + assert "supplier" in readme.lower() + assert "end-of-support" in readme.lower() + + +# --------------------------------------------------------------------------- +# Guards shared by every generated document +# --------------------------------------------------------------------------- + +ALL_PACKS = [ + pytest.param(lambda: _eu(_hf_sbom()), id="eu-hf"), + pytest.param(lambda: _eu(_local_sbom()), id="eu-local"), + pytest.param(lambda: _eu({"components": []}), id="eu-empty"), + pytest.param(lambda: _fda(_hf_sbom()), id="fda-hf"), + pytest.param(lambda: _fda(_local_sbom()), id="fda-local"), +] + + +@pytest.mark.parametrize("make", ALL_PACKS) +def test_no_generated_file_uses_verdict_language(make): + text = _all_text(make()).lower() + hits = [w for w in VERDICT_WORDS if w in text] + assert hits == [] + + +@pytest.mark.parametrize("make", ALL_PACKS) +def test_every_markdown_file_carries_disclaimer_and_template_version(make): + for name, text in make().items(): + if not name.endswith(".md"): + continue + assert "not legal advice" in text, name + assert comply.TEMPLATE_VERSION in text, name + # The disclaimer closes the document. + assert text.rstrip().endswith(comply.DISCLAIMER), name + + +def test_malformed_components_are_ignored_not_fatal(): + doc = {"components": ["junk", {"type": "machine-learning-model"}, None]} + assert comply.ANNEX_XI_FILE in _eu(doc) + assert comply.SUPPORT_FILE in _fda(doc) + + +def _section(text, start, end): + i = text.index(start) + j = text.index(end, i + len(start)) + return text[i:j] + + +# --------------------------------------------------------------------------- +# The CLI command +# --------------------------------------------------------------------------- + +def _restricted_tree(tmp_path): + from aisbom.mock_generator import create_mock_restricted_file + + src = tmp_path / "models" + src.mkdir() + create_mock_restricted_file(src) + return src + + +def test_cli_eu_ai_act_writes_the_pack(tmp_path): + src = _restricted_tree(tmp_path) + out = tmp_path / "pack" + result = runner.invoke(app, [ + "comply", str(src), "--framework", "eu-ai-act", "--output-dir", str(out), "--offline", + ]) + assert result.exit_code == 0, result.stdout + for name in (comply.ANNEX_XI_FILE, comply.ART53_FILE, comply.README_FILE, "sbom.json"): + assert (out / name).is_file(), name + sbom = json.loads((out / "sbom.json").read_text()) + assert sbom["bomFormat"] == "CycloneDX" + + +def test_cli_fda_writes_sbom_vex_and_support_table(tmp_path, monkeypatch): + from aisbom import osv + + def _boom(*a, **k): # --offline must keep OSV untouched + raise AssertionError("OSV queried in offline mode") + + monkeypatch.setattr(osv, "lookup_dependency_statements", _boom) + src = _restricted_tree(tmp_path) + (src / "requirements.txt").write_text("torch==2.10.0\n") + out = tmp_path / "pack" + result = runner.invoke(app, [ + "comply", str(src), "--framework", "fda", "--output-dir", str(out), "--offline", + ]) + assert result.exit_code == 0, result.stdout + for name in ("sbom.json", "sbom.openvex.json", "sbom.vex.cdx.json", + comply.SUPPORT_FILE, comply.README_FILE): + assert (out / name).is_file(), name + serial = json.loads((out / "sbom.json").read_text())["serialNumber"] + openvex = json.loads((out / "sbom.openvex.json").read_text()) + assert any(serial in json.dumps(s) for s in openvex["statements"]) + + +def test_cli_default_output_dir_is_named_for_the_framework(tmp_path, monkeypatch): + src = _restricted_tree(tmp_path) + monkeypatch.chdir(tmp_path) + result = runner.invoke(app, ["comply", str(src), "--framework", "fda", "--offline"]) + assert result.exit_code == 0, result.stdout + assert (tmp_path / "aisbom-comply-fda" / comply.README_FILE).is_file() + + +def test_cli_refuses_a_partial_scan(tmp_path, monkeypatch): + from aisbom import cli as cli_mod + + class _PartialScanner: + def __init__(self, *a, **kw): + pass + + def scan(self): + return { + "artifacts": [], + "dependencies": [], + "errors": [{"file": "broken.pt", "error": "unreadable"}], + } + + monkeypatch.setattr(cli_mod, "DeepScanner", _PartialScanner) + out = tmp_path / "pack" + result = runner.invoke(app, [ + "comply", str(tmp_path), "--framework", "eu-ai-act", "--output-dir", str(out), "--offline", + ]) + assert result.exit_code == 1 + assert "broken.pt" in result.stdout + assert not out.exists() + + +def test_cli_rejects_an_unusable_target(tmp_path): + result = runner.invoke(app, [ + "comply", str(tmp_path / "nope"), "--framework", "fda", "--offline", + ]) + assert result.exit_code == 1 + assert "Cannot build" in result.stdout + + +def test_cli_requires_a_framework(tmp_path): + result = runner.invoke(app, ["comply", str(tmp_path)]) + assert result.exit_code == 2 + + +def test_cli_rejects_an_unknown_framework(tmp_path): + result = runner.invoke(app, ["comply", str(tmp_path), "--framework", "hipaa"]) + assert result.exit_code == 2