diff --git a/.github/scripts/verify-sbom-scan-flag.sh b/.github/scripts/verify-sbom-scan-flag.sh new file mode 100755 index 00000000..7b97e587 --- /dev/null +++ b/.github/scripts/verify-sbom-scan-flag.sh @@ -0,0 +1,34 @@ +#!/bin/bash -ex + +set -o pipefail + +cd "$(git rev-parse --show-toplevel)" + +# +# Every package must explicitly set SBOM_DEEP_SCAN to "true" or "false" in +# its config.sh -- see docs on generate_sbom() in lib/common.sh. There is +# no default: a package that hasn't been classified yet must fail CI +# rather than silently ship without a CycloneDX sidecar or without an +# explicit decision that it doesn't need one. +# +unclassified=$(./query-packages.sh list -o name,sbom-deep-scan all | + awk -F'\t' '$2 == "none" { print $1 }') + +if [[ -n "$unclassified" ]]; then + echo "The following packages have not set SBOM_DEEP_SCAN (\"true\" or" \ + "\"false\") in their config.sh:" + echo "$unclassified" + echo + echo "Set it to \"true\" for 1st-party packages, so that the" \ + "third-party components they package internally (jars, npm" \ + "modules, Rust crates, ...) are included in the product's" \ + "aggregate SBOM." + echo "Set it to \"false\" for 3rd-party forks of Debian packages, and" \ + "for packages that are not included in a shipping product:" \ + "those are already covered as a flat pkg:deb component by" \ + "appliance-build's image-level scan, so a deep scan here would" \ + "add nothing." + exit 1 +fi + +echo "All packages have classified SBOM_DEEP_SCAN" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e527d7c7..f4f9a559 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,3 +29,8 @@ jobs: steps: - uses: actions/checkout@v1 - run: ./.github/scripts/verify-query-packages.sh + verify-sbom-scan-flag: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - run: ./.github/scripts/verify-sbom-scan-flag.sh \ No newline at end of file diff --git a/buildpkg.sh b/buildpkg.sh index 891afb2a..57d184d1 100755 --- a/buildpkg.sh +++ b/buildpkg.sh @@ -137,6 +137,9 @@ stage build logmust cd "$WORKDIR" stage store_build_info +logmust cd "$WORKDIR" +stage generate_sbom + logmust cd "$WORKDIR" stage post_build_checks diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md new file mode 100644 index 00000000..2538ff65 --- /dev/null +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -0,0 +1,350 @@ +# SBOM Per-Package Sidecar Generation — Design + +- **Date:** 2026-09-08 +- **Jira:** [DLPX-98872](https://perforce.atlassian.net/browse/DLPX-98872) +- **Epic:** [CP-13455](https://perforce.atlassian.net/browse/CP-13455) — CycloneDX SBOM for Delphix engine product images +- **Implements:** Phase 2 (CP-13465) of `appliance-build`'s + `docs/specs/2026-06-23-sbom-generation-implementation-plan.md` +- **Companion to (in the `appliance-build` repo):** `docs/specs/2026-06-15-sbom-generation-design.md` + (CP-13456, the overall design) and `docs/specs/2026-08-13-syft-cyclonedx-cli-provisioning-design.md` + (CP-13600, how `syft`/`cyclonedx-cli` get onto a build host) +- **Implemented in:** this change (DLPX-98872) — see *Implementation* below. + +## Builds on: Phase 1 (already implemented) + +Phase 1 — the `appliance-build` per-image base scan (CP-13464) — and its `syft`/ +`cyclonedx-cli` provisioning prerequisite (CP-13600) are implemented, each with an open PR: + +| Repo | PR | What it does | +|---|---|---| +| `syft` | [#1](https://github.com/delphix/syft/pull/1) | Packaging repo for `delphix-syft` (fetches the pinned upstream release binary) | +| `cyclonedx-cli` | [#1](https://github.com/delphix/cyclonedx-cli/pull/1) | Packaging repo for `delphix-cyclonedx-cli` (same pattern) | +| `linux-pkg` | [#414](https://github.com/delphix/linux-pkg/pull/414) | Adds `packages/syft/`, `packages/cyclonedx-cli/`, listed in `package-lists/build/main.pkgs` (this branch is rebased on top of it) | +| `appliance-build` | [#892](https://github.com/delphix/appliance-build/pull/892) | Installs both `.deb`s onto the build host in `build-ancillary-repository.sh`; new `95-generate-sbom.binary` hook runs `syft scan dir:binary --select-catalogers dpkg` + `cyclonedx-cli validate` to emit `-.cdx.json` | +| `devops-gate` | [#4717](https://github.com/delphix/devops-gate/pull/4717) | `appliance_build_stage0.groovy` best-effort-fetches and archives the per-image `.cdx.json` | + +Net effect: `syft` and `cyclonedx-cli` are already real, buildable `linux-pkg` packages, +already on every build host's `PATH` (build-host-only tooling — never shipped in the image), +and every image already gets a flat, dpkg-only base SBOM. **Phase 2 has no new tooling +dependency to add** — it only needs to *invoke* `syft`, which is already available. + +## Problem + +The Phase 1 base scan lists every installed `.deb` as a flat `pkg:deb` component. That is +correct and sufficient for 3rd-party Debian packages, but wrong for Delphix's own +first-party packages: `masking`, `virtualization`, `delphix-sso-app`, `containerized-masking`, +`windows-connector`, `zfs`, `ptools`, and `delphix-rust` all bundle third-party components +across ecosystems (jars, npm, wheels, Rust crates) that `dpkg` cannot see — dpkg only knows +the files *it* placed, not what's vendored inside a jar or statically linked into a Rust +binary. Those components are invisible to a vulnerability scanner today. + +Per the original design's key decision, this composition must be captured **upstream, in +`linux-pkg`**, at build time, where the package's own build artifacts are present — not +reconstructed later from a stripped, compiled `.deb`. + +## Design and implementation + +All of the following is implemented in this change, entirely within `linux-pkg` — no other +repo needs to change for Phase 2 (see *S3 upload* below for why). + +### 1. `SBOM_DEEP_SCAN` — per-package opt-in flag + +Mirrors the existing `MEND_SCAN_APPLICABLE` convention: a plain variable set in a package's +`config.sh`. Unlike `MEND_SCAN_APPLICABLE` (opt-in only, unset elsewhere), this flag is a real +tri-state — "must deep-scan" / "explicitly doesn't need it" / "nobody has classified this +yet" — enforced by the CI lint in §3, so **every** package's `config.sh` now has an explicit +`SBOM_DEEP_SCAN="true"` or `SBOM_DEEP_SCAN="false"` (see §5). + +```bash +# packages//config.sh +SBOM_DEEP_SCAN="true" +``` + +### 2. `query-packages.sh` — surface the field + +`query-packages.sh` has a closed, hardcoded field enum — no generic "query any config.sh +variable" mechanism exists. Two edits: + +- `ALL_OUTPUT_FIELDS`: appended `sbom-deep-scan`. +- `print_package()`'s `case "$field"` block, alongside the `mend-scan)` arm: + ```bash + sbom-deep-scan) outarray+=("${SBOM_DEEP_SCAN:-none}") ;; + ``` + +`-o`'s comma-delimited parsing and validation against `ALL_OUTPUT_FIELDS` then works for the +new field automatically. `./query-packages.sh list -o name,sbom-deep-scan all` is the +mechanism both the CI lint (§3) and `appliance-build`'s future Phase 3 consumer use to find +flagged packages. + +### 3. CI lint — every package must be classified + +No existing precedent to extend — `MEND_SCAN_APPLICABLE` is opt-in with zero enforcement +anywhere in `linux-pkg`. New script `.github/scripts/verify-sbom-scan-flag.sh`, wired as a +`verify-sbom-scan-flag` job in `.github/workflows/main.yml` alongside the existing +`verify-query-packages*` jobs: + +```bash +unclassified=$(./query-packages.sh list -o name,sbom-deep-scan all | + awk -F'\t' '$2 == "none" { print $1 }') +if [[ -n "$unclassified" ]]; then + echo "The following packages have not set SBOM_DEEP_SCAN (\"true\" or" \ + "\"false\") in their config.sh:" + echo "$unclassified" + exit 1 +fi +``` + +The one-time classification of all 40 packages (§5) lands in the same change, so this lint +never lands red. + +### 4. `generate_sbom()` — sidecar generation + +Modeled on `store_build_info()` — a default stage function in `lib/common.sh` that packages +don't need to override, gated on the new flag: + +```bash +# lib/common.sh +function generate_sbom() { + if [[ "$SBOM_DEEP_SCAN" != "true" ]]; then + return 0 + fi + + local debs=("$WORKDIR/artifacts/"*.deb) + if [[ ! -e "${debs[0]}" ]]; then + die "SBOM_DEEP_SCAN is set but no .deb was found in" \ + "'$WORKDIR/artifacts'" + fi + + # syft/cyclonedx-cli come from setup.sh's install_sbom_tools(), not + # from anything this package declares (see below). That install is + # best-effort, so check here rather than letting "command not + # found" surface mid-scan. + local tool + for tool in syft cyclonedx-cli; do + command -v "$tool" >/dev/null || die "'$tool' is not installed..." + done + + # One sidecar per .deb, not per package: a package that emits more + # than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux, + # etc.) gets one .deb.cdx.json per .deb -- a strict + # 1:1 mapping, no merging across a package's .deb(s). + local deb + for deb in "${debs[@]}"; do + local sbom_file deb_version + sbom_file="$WORKDIR/artifacts/$(basename "$deb").cdx.json" + deb_version="$(dpkg-deb -f "$deb" Version)" + SYFT_FILE_METADATA_SELECTION=none logmust syft scan "$deb" \ + --source-name "$PACKAGE" \ + --source-version "$deb_version" \ + -o "cyclonedx-json@1.6=$sbom_file" + + logmust cyclonedx-cli validate \ + --input-file "$sbom_file" \ + --input-format json \ + --input-version v1_6 \ + --fail-on-errors + done +} +``` + +Wired into `buildpkg.sh` as a new stage between the two that already bracket this point: + +```bash +logmust cd "$WORKDIR" +stage store_build_info +logmust cd "$WORKDIR" +stage generate_sbom # <-- new +logmust cd "$WORKDIR" +stage post_build_checks +``` + +`stage` silently skips undefined hooks, so this is inert for every package until +`generate_sbom` is defined — defining it once in `lib/common.sh`, gated internally on the +flag, means no package needs its own override for the baseline case (only per-ecosystem +overrides, out of scope for Phase 2, would override the function in a package's `config.sh`, +the same way packages already override `build()`). + +**Resolved — multiple `.deb`s per package:** a reviewer flagged that the original +merge-into-one-sidecar approach (below, kept here for history) doesn't give a clean 1:1 +mapping between a `.deb` and its BOM. Changed to: one `.deb.cdx.json` per +`.deb`, no merge, no `cyclonedx-cli merge` step at all — each `.deb`'s sidecar is fully +independent and filename-matched to it. This is a real divergence from the top-level +design doc (CP-13456), which called for "one package-level SBOM... associated with all +of that package's debs via `COMPONENTS`" — that assumption didn't survive review. Phase 3 +(`appliance-build`'s consumer, not yet built) will need to associate each `.deb` with its +own sidecar directly by filename, not go through a package-level indirection. + +*(For reference, the approach this replaced: scan each `.deb` into its own document, then +`cyclonedx-cli merge --output-version v1_6` them into a single `.cdx.json`. Two +real bugs were found and fixed while that was still in place, both still relevant to the +current code since they're not specific to the merge step: `syft`/`cyclonedx-cli` have to +actually be installed in the build container before use — see §6 — and Syft's default +file-metadata component needs suppressing via `SYFT_FILE_METADATA_SELECTION=none`.)* + +**Resolved — Syft's `.deb`-scan support:** confirmed working against the real `syft`/ +`cyclonedx-cli` binaries via an actual pre-push build (`delphix-sso-app`, `delphix-rust`) — +`syft scan ` scans a standalone `.deb` file directly, no extraction needed. + +### 5. Package classification + +Per the original design's "does this package bundle third-party composition" criterion (not +strictly "is it 1st-party" — see the `zfs` case), `SBOM_DEEP_SCAN="true"` is set for: + +| Package | Why | +|---|---| +| `masking` | Java/Gradle app bundling jars + npm frontend | +| `virtualization` | Java/Gradle app bundling jars + npm frontend | +| `delphix-sso-app` | Java/Gradle app | +| `containerized-masking` | Java/Gradle app | +| `windows-connector` | Java/Gradle app | +| `zfs` | OpenZFS fork bundling Delphix's Rust object agent (crates invisible to dpkg) | +| `ptools` | Rust | +| `delphix-rust` | Rust | + +The remaining 32 packages (kernel packages, `misc-debs`, `syft`/`cyclonedx-cli` themselves, +etc.) get `SBOM_DEEP_SCAN="false"` — plain 3rd-party forks or single-ecosystem tools already +fully represented by the Phase 1 flat `pkg:deb` component. `delphix-go` is a judgment call: +the top-level design's Tooling section separately calls out a possible Go override +(`cyclonedx-gomod`); defaulted to `"false"` here (baseline Syft-on-deb has a Go +binary-build-info cataloger that may already cover it) — revisit under Phase 4's evaluation +if gaps are found. + +### 6. Provisioning syft/cyclonedx-cli — generic build tooling, not a package dependency + +`generate_sbom()` is a **default hook**: defined once in `lib/common.sh` and inherited +unmodified by all 8 flagged packages (none of them override it, unlike e.g. `zfs`'s own +`build()`). The tooling it runs is therefore the hook's concern, not its callers' — so no +package's `config.sh` declares `syft`/`cyclonedx-cli` anywhere. They are installed with the +rest of the generic build tooling in `setup.sh`, which runs before every package build: + +```bash +logmust install_awscli +logmust install_sbom_tools # must follow install_awscli -- fetches from S3 +logmust install_shfmt +``` + +`install_sbom_tools()` (`lib/common.sh`) is modelled on the existing `install_awscli()`/ +`install_shfmt()` non-apt installers, with one difference: `delphix-syft`/ +`delphix-cyclonedx-cli` are Delphix-built `.deb`s with no apt source (the container's apt +sources are only the Ubuntu primary mirror plus the PPA secondary mirror), so they're +fetched from the same S3 location `fetch_dependencies()` uses — +`get_package_dependency_s3_url()` → `aws s3 cp --recursive` → `apt-get install ` +(apt, not dpkg, because `delphix-cyclonedx-cli` has real `libicu` dependencies). + +**Best-effort by design.** `setup.sh` is package-agnostic — it has no idea which package is +about to be built — so it cannot skip itself when the package being built *is* `syft` or +`cyclonedx-cli`. Hard-failing on a missing artifact would therefore break *every* build on +a branch where those two haven't been published yet, including their own. Instead a missing +artifact warns and continues, and `generate_sbom()` checks for the tools itself and fails +loudly — so only the builds that actually need them are affected. + +**Rejected alternative:** deriving `PACKAGE_DEPENDENCIES += "syft cyclonedx-cli"` from +`SBOM_DEEP_SCAN` in `load_package_config()`. That also keeps it out of each `config.sh`, +is scoped to just the 8 packages, and keeps the relationship visible to Jenkins's static +dependency graph (so `syft` batches before its dependents and a `syft` rebuild cascades to +them). It was implemented and working, but sits in the per-package dependency layer rather +than the generic installed-prior layer, which is not what review asked for. Noting the +trade-off explicitly: with the `setup.sh` approach, Jenkins no longer knows these packages +relate to `syft`/`cyclonedx-cli`, so that batching/rebuild-cascade behaviour is lost. + +### S3 upload — no new plumbing needed + +Confirmed by reading `devops-gate/jenkins/jobs/pipelines/linux_pkg_build_package.groovy`'s +`Publish` stage: it runs `aws s3 sync --delete --only-show-errors . ${env.S3_PACKAGE_PATH}` +from inside `dir("linux-pkg/workdir/artifacts")` — i.e., it uploads **the entire artifacts +directory verbatim**, whatever stages before it dropped there (`store_build_info`'s +`GIT_HASH`/`BUILD_INFO` files, the built `.deb`(s), etc.). `generate_sbom()` writing +`.cdx.json` into that same directory is automatically picked up by this existing +sync — **no `devops-gate` change required for Phase 2**, unlike Phase 1 which needed a new +fetch step in `appliance_build_stage0.groovy` because that pipeline wasn't already pulling +per-package artifact directories at all. + +## Architecture diagram + +``` ++---------------------------------------------------------------------------+ +| linux-pkg package build (buildpkg.sh) | +| | +| stage build --> $WORKDIR/artifacts/*.deb | +| stage store_build_info --> GIT_HASH, BUILD_INFO, ... | +| stage generate_sbom --> [only if SBOM_DEEP_SCAN="true"] | +| for each .deb: | +| syft scan -o cyclonedx-json@1.6 | +| --> .deb.cdx.json | +| cyclonedx-cli validate --fail-on-errors| +| (strict 1:1 .deb <-> BOM, no merging) | +| stage post_build_checks | ++------------------------------------+--------------------------------------+ + | + | devops-gate Publish stage: + | aws s3 sync (whole artifacts/ dir, + | unmodified from Phase 1) + v ++---------------------------------------------------------------------------+ +| S3: combined-packages/packages// | +| .deb | +| .deb.cdx.json <-- NEW: one per .deb, same prefix | +| GIT_HASH, BUILD_INFO, ... (unchanged) | ++---------------------------------------------------------------------------+ + | + | (Phase 3, not in scope here: + | appliance-build fetches each .deb's + | own sidecar by filename match) + v + [out of scope for Phase 2] +``` + +## Out of scope for Phase 2 + +- Per-ecosystem overrides (`cargo-cyclonedx` for Rust, `cyclonedx-gradle-plugin` for + Java) — Phase 4's evaluation decides if the Syft-on-deb baseline here is good enough + first. +- `appliance-build` fetching/merging these sidecars into the per-image document — that's + Phase 3 (CP-13466), which explicitly depends on this phase completing. +- DCT/Hyperscale sidecars — deferred in the top-level design, not filed as a story yet. +- The `devops-gate` publishing switch (CSV → CycloneDX) — unrelated to sidecar generation. + +## Implementation status + +- [x] `SBOM_DEEP_SCAN` classification added to all 40 packages' `config.sh` (§5). +- [x] `sbom-deep-scan` field added to `query-packages.sh` (§2). +- [x] `generate_sbom()` added to `lib/common.sh` and wired as `stage generate_sbom` in + `buildpkg.sh` (§4), producing a strict 1:1 `.deb` → `.cdx.json` mapping (§4). +- [x] CI lint added (§3), landed in the same change as the classification pass. +- [x] Verified locally: `verify-sbom-scan-flag.sh` and the existing + `verify-query-packages.sh` both pass; `shellcheck`/`shfmt` clean on every touched file. +- [x] Verified against a real build host: `delphix-sso-app` (single-`.deb`) and + `delphix-rust` (multi-`.deb`) pre-push builds both produce valid, schema-checked + CycloneDX 1.6 sidecars in S3, correctly named per `.deb`. +- [x] Confirmed compliant with reviewer feedback: `syft`, `cyclonedx-cli`, and all + `linux-kernel-*` packages are `SBOM_DEEP_SCAN="false"` (no SBOM generated for + build-host tooling or 3rd-party kernel forks), and the `.deb` ↔ `.cdx.json` mapping + is now strictly 1:1 with a shared filename prefix. + +## Bugs found and fixed during real-build testing + +Not caught by CI — only surfaced by actually running builds: + +1. **`syft`/`cyclonedx-cli` command not found.** Nothing installed them into the + `linux-pkg` build container (Phase 1 only solved this for the `appliance-build` host). + Fixed by installing them from `setup.sh` with the rest of the generic build tooling + (§6). An interim fix declared them as `PACKAGE_DEPENDENCIES` on each flagged package + instead; that was replaced per review feedback. +2. **Blank `--source-version`.** `$PACKAGE_VERSION` doesn't reliably survive to the + `generate_sbom` stage for packages that don't set it themselves (unlike + `syft`/`cyclonedx-cli`'s own `config.sh`). Fixed by reading the version back out of the + built `.deb` via `dpkg-deb -f "$deb" Version`. +3. **Stray Syft "file" component**, carrying file hashes and an absolute build-workspace + path, polluting the sidecar. Fixed with `SYFT_FILE_METADATA_SELECTION=none`, matching + `appliance-build`'s `95-generate-sbom.binary` hook. +4. **`cyclonedx-cli merge` defaulted to spec version 1.7**, failing the subsequent + `--input-version v1_6` validate call. Moot now that merging was removed entirely per + the 1:1-mapping change above, but the same lesson applies to any future + `cyclonedx-cli` invocation: pin `--output-version` explicitly, don't rely on its + default. + +## Follow-ups + +1. Phase 3 (`appliance-build`'s consumer) needs to key off the 1:1 `.deb` ↔ `.cdx.json` + filename mapping established here, not a package-level `COMPONENTS` indirection. +2. File as sub-tasks of DLPX-98872. \ No newline at end of file diff --git a/lib/common.sh b/lib/common.sh index 1faab9c7..4d65ab99 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -619,6 +619,66 @@ function install_shfmt() { echo "shfmt version $(shfmt -version) is installed." } +# +# Install the tooling generate_sbom() needs. That is a default hook, shared +# unmodified by every package that sets SBOM_DEEP_SCAN, so the tools it runs +# belong with the rest of the generic build tooling installed here rather than +# being declared as a dependency by each of those packages individually. +# +# Unlike everything else installed from setup.sh these are Delphix-built +# packages with no apt source, so they are fetched from the same S3 location +# fetch_dependencies() pulls a package's dependencies from, then installed by +# path. apt rather than dpkg, because delphix-cyclonedx-cli has real +# dependencies (libicu and the usual shared libraries) that dpkg will not +# resolve. +# +# Best-effort by design: this runs before *every* package build, including the +# builds of syft and cyclonedx-cli themselves, and on a branch where neither +# has been published yet there is nothing to fetch. Failing hard would break +# every build on such a branch rather than just SBOM generation, so a missing +# artifact warns and moves on; generate_sbom() checks for the tools itself and +# fails loudly, for the only builds that actually need them. +# +function install_sbom_tools() { + local pkg s3url tmpdir + local debs=() + + tmpdir="$(mktemp -d)" || die "Failed to create a temporary directory" + + for pkg in syft cyclonedx-cli; do + # + # Run in a command substitution so that a failure to resolve + # the URL (get_package_dependency_s3_url dies when a package + # has no published artifacts) leaves $s3url empty here instead + # of aborting setup. + # + s3url="$( + get_package_dependency_s3_url "$pkg" >/dev/null 2>&1 + echo "$_RET" + )" + if [[ -z "$s3url" ]]; then + echo "WARNING: no published artifacts found for '$pkg';" \ + "skipping the SBOM tooling install. Builds of" \ + "packages that set SBOM_DEEP_SCAN will fail until" \ + "'$pkg' has been built for this branch." + logmust rm -rf "$tmpdir" + return 0 + fi + + [[ "$s3url" != */ ]] && s3url="$s3url/" + logmust mkdir -p "$tmpdir/$pkg" + logmust aws s3 cp --only-show-errors --recursive \ + "$s3url" "$tmpdir/$pkg/" + done + + debs=("$tmpdir"/*/*.deb) + [[ -e "${debs[0]}" ]] || + die "No .deb found in the fetched syft/cyclonedx-cli artifacts" + + logmust install_pkgs "${debs[@]}" + logmust rm -rf "$tmpdir" +} + # # Install kernel headers packages for all target kernels. # The kernel packages are fetched from S3. @@ -1461,6 +1521,90 @@ function store_build_info() { fi } +# +# Generate a CycloneDX SBOM sidecar for each of this package's built +# .deb(s) by running Syft against it. Only packages that bundle +# third-party composition (jars, npm, wheels, Rust crates, ...) opt in +# via SBOM_DEEP_SCAN="true" in their config.sh -- everything else is +# left as a flat pkg:deb component by appliance-build's base chroot +# scan, so a deb scan here would add nothing. Each .cdx.json +# is dropped in $WORKDIR/artifacts/ alongside the .deb it describes -- +# a strict 1:1 mapping, no merging across a package's .deb(s) -- where +# it's picked up by the same S3 sync as every other build artifact, no +# separate upload path needed. +# +function generate_sbom() { + if [[ "$SBOM_DEEP_SCAN" != "true" ]]; then + return 0 + fi + + local debs=("$WORKDIR/artifacts/"*.deb) + if [[ ! -e "${debs[0]}" ]]; then + die "SBOM_DEEP_SCAN is set but no .deb was found in" \ + "'$WORKDIR/artifacts'" + fi + + # + # syft/cyclonedx-cli are part of the generic build tooling installed + # by setup.sh (install_sbom_tools()) before any package is built, + # rather than something each SBOM_DEEP_SCAN package declares for + # itself -- this is a default hook, so what it needs is its own + # concern, not its callers'. That install is best-effort, so check + # here rather than letting "command not found" surface from the + # middle of a scan. + # + local tool + for tool in syft cyclonedx-cli; do + command -v "$tool" >/dev/null || + die "'$tool' is not installed, so no SBOM can be" \ + "generated for '$PACKAGE'. It is provisioned by" \ + "install_sbom_tools() in setup.sh; check that" \ + "run's output for why it was skipped." + done + + # + # One sidecar per .deb, not per package: a package that emits more + # than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux, + # etc.) gets one .deb.cdx.json per .deb, each a + # standalone document scoped to that .deb alone. No merging across + # .debs -- keeps a strict 1:1 mapping between a .deb and its BOM, + # with the .deb's own filename as the common prefix. + # + local deb + for deb in "${debs[@]}"; do + local sbom_file deb_version + sbom_file="$WORKDIR/artifacts/$(basename "$deb").cdx.json" + # + # Read the version back out of the .deb itself, rather than + # relying on $PACKAGE_VERSION: by this point in the build, + # $PACKAGE_VERSION may no longer hold the final, + # revision-suffixed version set_changelog() wrote into the + # package (e.g. it's empty for packages that don't set it + # explicitly themselves, unlike syft/cyclonedx-cli's own + # config.sh). dpkg-deb reads the actual, authoritative + # version of the artifact being scanned. + # + deb_version="$(dpkg-deb -f "$deb" Version)" + # + # SYFT_FILE_METADATA_SELECTION=none suppresses Syft's default + # per-file "file" component (with SHA-1/SHA-256 hashes and the + # absolute build-workspace path baked in) -- noise that doesn't + # belong in a per-deb sidecar. Same reasoning as + # appliance-build's 95-generate-sbom.binary hook. + # + SYFT_FILE_METADATA_SELECTION=none logmust syft scan "$deb" \ + --source-name "$PACKAGE" \ + --source-version "$deb_version" \ + -o "cyclonedx-json@1.6=$sbom_file" + + logmust cyclonedx-cli validate \ + --input-file "$sbom_file" \ + --input-format json \ + --input-version v1_6 \ + --fail-on-errors + done +} + function set_secret_build_args() { _SECRET_BUILD_ARGS=() diff --git a/packages/bcc/config.sh b/packages/bcc/config.sh index 047b6922..1094b594 100644 --- a/packages/bcc/config.sh +++ b/packages/bcc/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bcc.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL=https://github.com/iovisor/bcc.git UPSTREAM_GIT_BRANCH=master diff --git a/packages/challenge-response/config.sh b/packages/challenge-response/config.sh index 997f96f8..6c29a67d 100644 --- a/packages/challenge-response/config.sh +++ b/packages/challenge-response/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/challenge-response.git" +SBOM_DEEP_SCAN="false" function prepare() { install_build_deps_from_control_file diff --git a/packages/cloud-init/config.sh b/packages/cloud-init/config.sh index dd720656..aab2ad00 100644 --- a/packages/cloud-init/config.sh +++ b/packages/cloud-init/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/cloud-init.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE=cloud-init diff --git a/packages/connstat/config.sh b/packages/connstat/config.sh index 4e50e1b4..08ea16ed 100644 --- a/packages/connstat/config.sh +++ b/packages/connstat/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/connstat.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="@linux-kernel dwarves" function prepare() { diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh index 36b06aca..d1cff120 100644 --- a/packages/containerized-masking/config.sh +++ b/packages/containerized-masking/config.sh @@ -40,6 +40,8 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" # PACKAGE_NEEDS_DOCKER="true" MEND_SCAN_APPLICABLE="true" +SBOM_DEEP_SCAN="true" + MEND_SCAN_IMAGES="'delphix-masking-proxy', 'delphix-masking-database', 'delphix-masking-app'" SKIP_COPYRIGHTS_CHECK=true diff --git a/packages/crash-python/config.sh b/packages/crash-python/config.sh index 495597fc..ac839c98 100644 --- a/packages/crash-python/config.sh +++ b/packages/crash-python/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/crash-python.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust install_build_deps_from_control_file diff --git a/packages/crypt-blowfish/config.sh b/packages/crypt-blowfish/config.sh index 04244a4b..b1f26638 100644 --- a/packages/crypt-blowfish/config.sh +++ b/packages/crypt-blowfish/config.sh @@ -17,6 +17,8 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/crypt-blowfish.git" +SBOM_DEEP_SCAN="false" + SKIP_COPYRIGHTS_CHECK=true function build() { diff --git a/packages/cyclonedx-cli/config.sh b/packages/cyclonedx-cli/config.sh index 3ee39875..ed56ae59 100755 --- a/packages/cyclonedx-cli/config.sh +++ b/packages/cyclonedx-cli/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/cyclonedx-cli.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-go/config.sh b/packages/delphix-go/config.sh index f9926295..9dfc17d1 100755 --- a/packages/delphix-go/config.sh +++ b/packages/delphix-go/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-go.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-kernel/config.sh b/packages/delphix-kernel/config.sh index 125836ff..c556cc25 100644 --- a/packages/delphix-kernel/config.sh +++ b/packages/delphix-kernel/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-kernel.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="@linux-kernel" function prepare() { diff --git a/packages/delphix-platform/config.sh b/packages/delphix-platform/config.sh index 42e4ff4c..c83aa809 100644 --- a/packages/delphix-platform/config.sh +++ b/packages/delphix-platform/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-platform.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust cd "$WORKDIR/repo" diff --git a/packages/delphix-rust/config.sh b/packages/delphix-rust/config.sh index 4f1f06c0..c1613a2b 100755 --- a/packages/delphix-rust/config.sh +++ b/packages/delphix-rust/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-rust.git" +SBOM_DEEP_SCAN="true" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 6d1a9c05..480f79d0 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -19,6 +19,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/saml-app.git" MEND_SCAN_APPLICABLE="true" MEND_SCAN_USING_SUDO="true" +SBOM_DEEP_SCAN="true" function prepare() { logmust install_pkgs openjdk-17-jdk-headless: diff --git a/packages/docker-python-image/config.sh b/packages/docker-python-image/config.sh index b80e6172..9379b26a 100644 --- a/packages/docker-python-image/config.sh +++ b/packages/docker-python-image/config.sh @@ -17,6 +17,8 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/docker-python-image.git" +SBOM_DEEP_SCAN="false" + # # debian/rules' override_dh_install runs 'docker pull' to fetch the python # image it repackages, so the build needs a docker daemon. The build container diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh index 0387a3a7..3f29a512 100644 --- a/packages/drgn/config.sh +++ b/packages/drgn/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/drgn.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="libkdumpfile" UPSTREAM_GIT_URL="https://github.com/osandov/drgn.git" diff --git a/packages/dwarves/config.sh b/packages/dwarves/config.sh index 46965c54..86cfcc66 100644 --- a/packages/dwarves/config.sh +++ b/packages/dwarves/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dwarves.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL="https://github.com/acmel/dwarves.git" UPSTREAM_GIT_BRANCH="master" diff --git a/packages/fluentd-gems/config.sh b/packages/fluentd-gems/config.sh index b511f1bb..2110330b 100644 --- a/packages/fluentd-gems/config.sh +++ b/packages/fluentd-gems/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/fluentd-gems.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/gdb-python/config.sh b/packages/gdb-python/config.sh index 014215f0..c36d22dd 100644 --- a/packages/gdb-python/config.sh +++ b/packages/gdb-python/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/gdb-python.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="libkdumpfile" function prepare() { diff --git a/packages/grub2/config.sh b/packages/grub2/config.sh index 82452184..e280c484 100644 --- a/packages/grub2/config.sh +++ b/packages/grub2/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none +SBOM_DEEP_SCAN="false" SKIP_COPYRIGHTS_CHECK=true URI="s3://release-de-images/internal-artifacts/2025.3.0.1/1.0.53/input-artifacts/combined-packages/packages/grub2" diff --git a/packages/host-jdks/config.sh b/packages/host-jdks/config.sh index 5aec9a30..d0b93c80 100755 --- a/packages/host-jdks/config.sh +++ b/packages/host-jdks/config.sh @@ -23,6 +23,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/host-jdks.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/libkdumpfile/config.sh b/packages/libkdumpfile/config.sh index d1dd6f52..91976030 100644 --- a/packages/libkdumpfile/config.sh +++ b/packages/libkdumpfile/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/libkdumpfile.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL="https://codeberg.org/ptesarik/libkdumpfile.git" UPSTREAM_GIT_BRANCH="tip" diff --git a/packages/linux-kernel-aws/config.sh b/packages/linux-kernel-aws/config.sh index b2596d2f..1f00b06c 100644 --- a/packages/linux-kernel-aws/config.sh +++ b/packages/linux-kernel-aws/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-azure/config.sh b/packages/linux-kernel-azure/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-azure/config.sh +++ b/packages/linux-kernel-azure/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-gcp/config.sh b/packages/linux-kernel-gcp/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-gcp/config.sh +++ b/packages/linux-kernel-gcp/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-generic/config.sh b/packages/linux-kernel-generic/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-generic/config.sh +++ b/packages/linux-kernel-generic/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-oracle/config.sh b/packages/linux-kernel-oracle/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-oracle/config.sh +++ b/packages/linux-kernel-oracle/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/makedumpfile/config.sh b/packages/makedumpfile/config.sh index 8eb407a5..3ad8172d 100644 --- a/packages/makedumpfile/config.sh +++ b/packages/makedumpfile/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/makedumpfile.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE="makedumpfile" diff --git a/packages/masking/config.sh b/packages/masking/config.sh index 92163bdb..2229e02d 100644 --- a/packages/masking/config.sh +++ b/packages/masking/config.sh @@ -20,6 +20,7 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" MEND_SCAN_APPLICABLE="true" +SBOM_DEEP_SCAN="true" function prepare() { logmust read_list "$WORKDIR/repo/packaging/build-dependencies" diff --git a/packages/misc-debs/config.sh b/packages/misc-debs/config.sh index 19fa7c7d..32bb43bd 100644 --- a/packages/misc-debs/config.sh +++ b/packages/misc-debs/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none +SBOM_DEEP_SCAN="false" SKIP_COPYRIGHTS_CHECK=true # diff --git a/packages/nfs-utils/config.sh b/packages/nfs-utils/config.sh index 15e3c725..639097d6 100644 --- a/packages/nfs-utils/config.sh +++ b/packages/nfs-utils/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/nfs-utils.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE=nfs-utils diff --git a/packages/performance-diagnostics/config.sh b/packages/performance-diagnostics/config.sh index 4733a8c9..6d666c36 100644 --- a/packages/performance-diagnostics/config.sh +++ b/packages/performance-diagnostics/config.sh @@ -18,6 +18,7 @@ # DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/performance-diagnostics.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust install_build_deps_from_control_file diff --git a/packages/ptools/config.sh b/packages/ptools/config.sh index 459cc4ca..54aef875 100644 --- a/packages/ptools/config.sh +++ b/packages/ptools/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/ptools.git" +SBOM_DEEP_SCAN="true" function prepare() { logmust install_pkgs cargo devscripts diff --git a/packages/python-rtslib-fb/config.sh b/packages/python-rtslib-fb/config.sh index 024c025e..99c334ef 100644 --- a/packages/python-rtslib-fb/config.sh +++ b/packages/python-rtslib-fb/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/python-rtslib-fb.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE=python-rtslib-fb diff --git a/packages/savedump/config.sh b/packages/savedump/config.sh index 1e9bdda2..ba291b44 100644 --- a/packages/savedump/config.sh +++ b/packages/savedump/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/savedump.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust install_build_deps_from_control_file diff --git a/packages/sdb/config.sh b/packages/sdb/config.sh index 3b36fed4..119abc18 100644 --- a/packages/sdb/config.sh +++ b/packages/sdb/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/sdb.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL="https://github.com/sdimitro/sdb.git" UPSTREAM_GIT_BRANCH="develop" diff --git a/packages/syft/config.sh b/packages/syft/config.sh index 987ee8f6..d4d40573 100755 --- a/packages/syft/config.sh +++ b/packages/syft/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/syft.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/targetcli-fb/config.sh b/packages/targetcli-fb/config.sh index abdc2ea3..3841b56b 100644 --- a/packages/targetcli-fb/config.sh +++ b/packages/targetcli-fb/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/targetcli-fb.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="python-rtslib-fb" UPSTREAM_SOURCE_PACKAGE=targetcli-fb diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh index e7fe6b3a..171b111a 100644 --- a/packages/virtualization/config.sh +++ b/packages/virtualization/config.sh @@ -27,6 +27,7 @@ PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks" # PACKAGE_NEEDS_DOCKER="true" MEND_SCAN_APPLICABLE="true" +SBOM_DEEP_SCAN="true" function prepare() { logmust read_list "$WORKDIR/repo/appliance/packaging/build-dependencies" diff --git a/packages/windows-connector/config.sh b/packages/windows-connector/config.sh index 40479cd2..185083da 100644 --- a/packages/windows-connector/config.sh +++ b/packages/windows-connector/config.sh @@ -25,6 +25,7 @@ # DEFAULT_PACKAGE_GIT_URL="none" SKIP_COPYRIGHTS_CHECK=true +SBOM_DEEP_SCAN="true" function fetch() { PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index bd1edce5..de13dcb1 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/zfs.git" PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves" +SBOM_DEEP_SCAN="true" UPSTREAM_GIT_URL="https://github.com/openzfs/zfs.git" UPSTREAM_GIT_BRANCH="master" diff --git a/query-packages.sh b/query-packages.sh index d92c7c23..612f70ed 100755 --- a/query-packages.sh +++ b/query-packages.sh @@ -25,7 +25,7 @@ source "$TOP/lib/common.sh" # export LOGGING=false -ALL_OUTPUT_FIELDS=(name git-url dependencies can-update mend-scan mend-scan-images mend-scan-using-sudo) +ALL_OUTPUT_FIELDS=(name git-url dependencies can-update mend-scan mend-scan-images mend-scan-using-sudo sbom-deep-scan) function usage() { local output_fields="${ALL_OUTPUT_FIELDS[*]}" @@ -73,6 +73,7 @@ function print_package() { mend-scan) outarray+=("${MEND_SCAN_APPLICABLE:-none}") ;; mend-scan-images) outarray+=("${MEND_SCAN_IMAGES:-none}") ;; mend-scan-using-sudo) outarray+=("${MEND_SCAN_USING_SUDO:-none}") ;; + sbom-deep-scan) outarray+=("${SBOM_DEEP_SCAN:-none}") ;; dependencies) for dep in $PACKAGE_DEPENDENCIES; do check_package_exists "$dep" diff --git a/setup.sh b/setup.sh index cb8248b5..c0a00815 100755 --- a/setup.sh +++ b/setup.sh @@ -263,6 +263,12 @@ function install_awscli() { logmust install_awscli +# +# Must follow install_awscli: the syft/cyclonedx-cli packages it installs are +# fetched from S3, so the aws CLI has to be present first. +# +logmust install_sbom_tools + logmust install_shfmt logmust git config --global user.email "eng@delphix.com"