From ee451b24a58e987fb804793ee24d5e84eaba3ebd Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Wed, 9 Sep 2026 13:35:30 +0530 Subject: [PATCH 1/8] DLPX-98872 linux-pkg: generate per-package CycloneDX SBOM via Syft deb scan and publish S3 sidecar PR URL: https://www.github.com/delphix/linux-pkg/pull/418 --- .github/scripts/verify-sbom-scan-flag.sh | 24 ++ .github/workflows/main.yml | 5 + buildpkg.sh | 3 + ...6-09-08-sbom-per-package-sidecar-design.md | 279 ++++++++++++++++++ lib/common.sh | 61 ++++ packages/bcc/config.sh | 1 + packages/challenge-response/config.sh | 1 + packages/cloud-init/config.sh | 1 + packages/connstat/config.sh | 1 + packages/containerized-masking/config.sh | 1 + packages/crash-python/config.sh | 1 + packages/crypt-blowfish/config.sh | 2 + packages/cyclonedx-cli/config.sh | 1 + packages/delphix-go/config.sh | 1 + packages/delphix-kernel/config.sh | 1 + packages/delphix-platform/config.sh | 1 + packages/delphix-rust/config.sh | 1 + packages/delphix-sso-app/config.sh | 1 + packages/docker-python-image/config.sh | 4 +- packages/drgn/config.sh | 1 + packages/dwarves/config.sh | 1 + packages/fluentd-gems/config.sh | 1 + packages/gdb-python/config.sh | 1 + packages/grub2/config.sh | 1 + packages/host-jdks/config.sh | 1 + packages/libkdumpfile/config.sh | 1 + packages/linux-kernel-aws/config.sh | 3 + packages/linux-kernel-azure/config.sh | 3 + packages/linux-kernel-gcp/config.sh | 3 + packages/linux-kernel-generic/config.sh | 3 + packages/linux-kernel-oracle/config.sh | 3 + packages/makedumpfile/config.sh | 1 + packages/masking/config.sh | 1 + packages/misc-debs/config.sh | 1 + packages/nfs-utils/config.sh | 1 + packages/performance-diagnostics/config.sh | 1 + packages/ptools/config.sh | 1 + packages/python-rtslib-fb/config.sh | 1 + packages/savedump/config.sh | 1 + packages/sdb/config.sh | 1 + packages/syft/config.sh | 1 + packages/targetcli-fb/config.sh | 1 + packages/virtualization/config.sh | 1 + packages/windows-connector/config.sh | 1 + packages/zfs/config.sh | 1 + query-packages.sh | 3 +- 46 files changed, 427 insertions(+), 2 deletions(-) create mode 100755 .github/scripts/verify-sbom-scan-flag.sh create mode 100644 docs/specs/2026-09-08-sbom-per-package-sidecar-design.md diff --git a/.github/scripts/verify-sbom-scan-flag.sh b/.github/scripts/verify-sbom-scan-flag.sh new file mode 100755 index 00000000..94691ddd --- /dev/null +++ b/.github/scripts/verify-sbom-scan-flag.sh @@ -0,0 +1,24 @@ +#!/bin/bash -ex + +set -o pipefail + +cd "$(git rev-parse --show-toplevel)" + +# +# Every package must explicitly set SBOM_DEEP_SCAN to "true" or "false" in +# its config.sh -- see docs on generate_sbom() in lib/common.sh. There is +# no default: a package that hasn't been classified yet must fail CI +# rather than silently ship without a CycloneDX sidecar or without an +# explicit decision that it doesn't need one. +# +unclassified=$(./query-packages.sh list -o name,sbom-deep-scan all | + awk -F'\t' '$2 == "none" { print $1 }') + +if [[ -n "$unclassified" ]]; then + echo "The following packages have not set SBOM_DEEP_SCAN (\"true\" or" \ + "\"false\") in their config.sh:" + echo "$unclassified" + exit 1 +fi + +echo "All packages have classified SBOM_DEEP_SCAN" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e527d7c7..f4f9a559 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,3 +29,8 @@ jobs: steps: - uses: actions/checkout@v1 - run: ./.github/scripts/verify-query-packages.sh + verify-sbom-scan-flag: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - run: ./.github/scripts/verify-sbom-scan-flag.sh \ No newline at end of file diff --git a/buildpkg.sh b/buildpkg.sh index 891afb2a..57d184d1 100755 --- a/buildpkg.sh +++ b/buildpkg.sh @@ -137,6 +137,9 @@ stage build logmust cd "$WORKDIR" stage store_build_info +logmust cd "$WORKDIR" +stage generate_sbom + logmust cd "$WORKDIR" stage post_build_checks diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md new file mode 100644 index 00000000..7e97bccc --- /dev/null +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -0,0 +1,279 @@ +# SBOM Per-Package Sidecar Generation — Design + +- **Date:** 2026-09-08 +- **Jira:** [DLPX-98872](https://perforce.atlassian.net/browse/DLPX-98872) +- **Epic:** [CP-13455](https://perforce.atlassian.net/browse/CP-13455) — CycloneDX SBOM for Delphix engine product images +- **Implements:** Phase 2 (CP-13465) of `appliance-build`'s + `docs/specs/2026-06-23-sbom-generation-implementation-plan.md` +- **Companion to (in the `appliance-build` repo):** `docs/specs/2026-06-15-sbom-generation-design.md` + (CP-13456, the overall design) and `docs/specs/2026-08-13-syft-cyclonedx-cli-provisioning-design.md` + (CP-13600, how `syft`/`cyclonedx-cli` get onto a build host) +- **Implemented in:** this change (DLPX-98872) — see *Implementation* below. + +## Builds on: Phase 1 (already implemented) + +Phase 1 — the `appliance-build` per-image base scan (CP-13464) — and its `syft`/ +`cyclonedx-cli` provisioning prerequisite (CP-13600) are implemented, each with an open PR: + +| Repo | PR | What it does | +|---|---|---| +| `syft` | [#1](https://github.com/delphix/syft/pull/1) | Packaging repo for `delphix-syft` (fetches the pinned upstream release binary) | +| `cyclonedx-cli` | [#1](https://github.com/delphix/cyclonedx-cli/pull/1) | Packaging repo for `delphix-cyclonedx-cli` (same pattern) | +| `linux-pkg` | [#414](https://github.com/delphix/linux-pkg/pull/414) | Adds `packages/syft/`, `packages/cyclonedx-cli/`, listed in `package-lists/build/main.pkgs` (this branch is rebased on top of it) | +| `appliance-build` | [#892](https://github.com/delphix/appliance-build/pull/892) | Installs both `.deb`s onto the build host in `build-ancillary-repository.sh`; new `95-generate-sbom.binary` hook runs `syft scan dir:binary --select-catalogers dpkg` + `cyclonedx-cli validate` to emit `-.cdx.json` | +| `devops-gate` | [#4717](https://github.com/delphix/devops-gate/pull/4717) | `appliance_build_stage0.groovy` best-effort-fetches and archives the per-image `.cdx.json` | + +Net effect: `syft` and `cyclonedx-cli` are already real, buildable `linux-pkg` packages, +already on every build host's `PATH` (build-host-only tooling — never shipped in the image), +and every image already gets a flat, dpkg-only base SBOM. **Phase 2 has no new tooling +dependency to add** — it only needs to *invoke* `syft`, which is already available. + +## Problem + +The Phase 1 base scan lists every installed `.deb` as a flat `pkg:deb` component. That is +correct and sufficient for 3rd-party Debian packages, but wrong for Delphix's own +first-party packages: `masking`, `virtualization`, `delphix-sso-app`, `containerized-masking`, +`windows-connector`, `zfs`, `ptools`, and `delphix-rust` all bundle third-party components +across ecosystems (jars, npm, wheels, Rust crates) that `dpkg` cannot see — dpkg only knows +the files *it* placed, not what's vendored inside a jar or statically linked into a Rust +binary. Those components are invisible to a vulnerability scanner today. + +Per the original design's key decision, this composition must be captured **upstream, in +`linux-pkg`**, at build time, where the package's own build artifacts are present — not +reconstructed later from a stripped, compiled `.deb`. + +## Design and implementation + +All of the following is implemented in this change, entirely within `linux-pkg` — no other +repo needs to change for Phase 2 (see *S3 upload* below for why). + +### 1. `SBOM_DEEP_SCAN` — per-package opt-in flag + +Mirrors the existing `MEND_SCAN_APPLICABLE` convention: a plain variable set in a package's +`config.sh`. Unlike `MEND_SCAN_APPLICABLE` (opt-in only, unset elsewhere), this flag is a real +tri-state — "must deep-scan" / "explicitly doesn't need it" / "nobody has classified this +yet" — enforced by the CI lint in §3, so **every** package's `config.sh` now has an explicit +`SBOM_DEEP_SCAN="true"` or `SBOM_DEEP_SCAN="false"` (see §5). + +```bash +# packages//config.sh +SBOM_DEEP_SCAN="true" +``` + +### 2. `query-packages.sh` — surface the field + +`query-packages.sh` has a closed, hardcoded field enum — no generic "query any config.sh +variable" mechanism exists. Two edits: + +- `ALL_OUTPUT_FIELDS`: appended `sbom-deep-scan`. +- `print_package()`'s `case "$field"` block, alongside the `mend-scan)` arm: + ```bash + sbom-deep-scan) outarray+=("${SBOM_DEEP_SCAN:-none}") ;; + ``` + +`-o`'s comma-delimited parsing and validation against `ALL_OUTPUT_FIELDS` then works for the +new field automatically. `./query-packages.sh list -o name,sbom-deep-scan all` is the +mechanism both the CI lint (§3) and `appliance-build`'s future Phase 3 consumer use to find +flagged packages. + +### 3. CI lint — every package must be classified + +No existing precedent to extend — `MEND_SCAN_APPLICABLE` is opt-in with zero enforcement +anywhere in `linux-pkg`. New script `.github/scripts/verify-sbom-scan-flag.sh`, wired as a +`verify-sbom-scan-flag` job in `.github/workflows/main.yml` alongside the existing +`verify-query-packages*` jobs: + +```bash +unclassified=$(./query-packages.sh list -o name,sbom-deep-scan all | + awk -F'\t' '$2 == "none" { print $1 }') +if [[ -n "$unclassified" ]]; then + echo "The following packages have not set SBOM_DEEP_SCAN (\"true\" or" \ + "\"false\") in their config.sh:" + echo "$unclassified" + exit 1 +fi +``` + +The one-time classification of all 40 packages (§5) lands in the same change, so this lint +never lands red. + +### 4. `generate_sbom()` — sidecar generation + +Modeled on `store_build_info()` — a default stage function in `lib/common.sh` that packages +don't need to override, gated on the new flag: + +```bash +# lib/common.sh +function generate_sbom() { + if [[ "$SBOM_DEEP_SCAN" != "true" ]]; then + return 0 + fi + + local debs=("$WORKDIR/artifacts/"*.deb) + if [[ ! -e "${debs[0]}" ]]; then + die "SBOM_DEEP_SCAN is set but no .deb was found in" \ + "'$WORKDIR/artifacts'" + fi + + local sbom_file="$WORKDIR/artifacts/$PACKAGE.cdx.json" + local sbom_scratch_dir + sbom_scratch_dir="$(logmust mktemp -d)" + + # A package can emit more than one .deb from a single build (e.g. + # "zfs" splits into zfs-dkms, zfsutils-linux, etc.) -- scan each one + # into its own document, then merge into a single sidecar so there's + # exactly one .cdx.json per package, matching how + # appliance-build associates a sidecar to a package via COMPONENTS. + local deb sbom_parts=() + for deb in "${debs[@]}"; do + local part + part="$sbom_scratch_dir/$(basename "$deb").cdx.json" + logmust syft scan "$deb" \ + --source-name "$PACKAGE" \ + --source-version "$PACKAGE_VERSION" \ + -o "cyclonedx-json@1.6=$part" + sbom_parts+=("$part") + done + + if [[ ${#sbom_parts[@]} -eq 1 ]]; then + logmust cp "${sbom_parts[0]}" "$sbom_file" + else + logmust cyclonedx-cli merge \ + --input-files "${sbom_parts[@]}" \ + --output-format json \ + --output-file "$sbom_file" + fi + logmust rm -rf "$sbom_scratch_dir" + + logmust cyclonedx-cli validate \ + --input-file "$sbom_file" \ + --input-format json \ + --input-version v1_6 \ + --fail-on-errors +} +``` + +Wired into `buildpkg.sh` as a new stage between the two that already bracket this point: + +```bash +logmust cd "$WORKDIR" +stage store_build_info +logmust cd "$WORKDIR" +stage generate_sbom # <-- new +logmust cd "$WORKDIR" +stage post_build_checks +``` + +`stage` silently skips undefined hooks, so this is inert for every package until +`generate_sbom` is defined — defining it once in `lib/common.sh`, gated internally on the +flag, means no package needs its own override for the baseline case (only per-ecosystem +overrides, out of scope for Phase 2, would override the function in a package's `config.sh`, +the same way packages already override `build()`). + +**Resolved — multiple `.deb`s per package:** handled above via per-deb scan + `cyclonedx-cli +merge` into one sidecar. Verify against a real `zfs` build before merging (see *Follow-ups*). + +**Open — Syft's `.deb`-scan support:** not yet verified against the real `syft`/ +`cyclonedx-cli` binaries in this environment (neither is installed here). Confirm the pinned +`delphix-syft` version (`SYFT_VERSION` in the `syft` packaging repo) supports scanning a +standalone `.deb` file as a source (Phase 1 only exercises Syft against a directory/rootfs, +not a single `.deb`). If unsupported, fall back to extracting the `.deb`'s `data.tar.*` into +a temp dir first and scanning that directory instead. + +### 5. Package classification + +Per the original design's "does this package bundle third-party composition" criterion (not +strictly "is it 1st-party" — see the `zfs` case), `SBOM_DEEP_SCAN="true"` is set for: + +| Package | Why | +|---|---| +| `masking` | Java/Gradle app bundling jars + npm frontend | +| `virtualization` | Java/Gradle app bundling jars + npm frontend | +| `delphix-sso-app` | Java/Gradle app | +| `containerized-masking` | Java/Gradle app | +| `windows-connector` | Java/Gradle app | +| `zfs` | OpenZFS fork bundling Delphix's Rust object agent (crates invisible to dpkg) | +| `ptools` | Rust | +| `delphix-rust` | Rust | + +The remaining 32 packages (kernel packages, `misc-debs`, `syft`/`cyclonedx-cli` themselves, +etc.) get `SBOM_DEEP_SCAN="false"` — plain 3rd-party forks or single-ecosystem tools already +fully represented by the Phase 1 flat `pkg:deb` component. `delphix-go` is a judgment call: +the top-level design's Tooling section separately calls out a possible Go override +(`cyclonedx-gomod`); defaulted to `"false"` here (baseline Syft-on-deb has a Go +binary-build-info cataloger that may already cover it) — revisit under Phase 4's evaluation +if gaps are found. + +### S3 upload — no new plumbing needed + +Confirmed by reading `devops-gate/jenkins/jobs/pipelines/linux_pkg_build_package.groovy`'s +`Publish` stage: it runs `aws s3 sync --delete --only-show-errors . ${env.S3_PACKAGE_PATH}` +from inside `dir("linux-pkg/workdir/artifacts")` — i.e., it uploads **the entire artifacts +directory verbatim**, whatever stages before it dropped there (`store_build_info`'s +`GIT_HASH`/`BUILD_INFO` files, the built `.deb`(s), etc.). `generate_sbom()` writing +`.cdx.json` into that same directory is automatically picked up by this existing +sync — **no `devops-gate` change required for Phase 2**, unlike Phase 1 which needed a new +fetch step in `appliance_build_stage0.groovy` because that pipeline wasn't already pulling +per-package artifact directories at all. + +## Architecture diagram + +``` ++---------------------------------------------------------------------------+ +| linux-pkg package build (buildpkg.sh) | +| | +| stage build --> $WORKDIR/artifacts/*.deb | +| stage store_build_info --> GIT_HASH, BUILD_INFO, ... | +| stage generate_sbom --> [only if SBOM_DEEP_SCAN="true"] | +| syft scan -o cyclonedx-json@1.6 | +| (merge via cyclonedx-cli if >1 .deb) | +| --> $WORKDIR/artifacts/.cdx.json | +| cyclonedx-cli validate --fail-on-errors | +| stage post_build_checks | ++------------------------------------+--------------------------------------+ + | + | devops-gate Publish stage: + | aws s3 sync (whole artifacts/ dir, + | unmodified from Phase 1) + v ++---------------------------------------------------------------------------+ +| S3: combined-packages/packages// | +| *.deb | +| .cdx.json <-- NEW: sidecar, sits beside the deb(s) | +| GIT_HASH, BUILD_INFO, ... (unchanged) | ++---------------------------------------------------------------------------+ + | + | (Phase 3, not in scope here: + | appliance-build fetches + merges + | this sidecar into the Phase 1 base) + v + [out of scope for Phase 2] +``` + +## Out of scope for Phase 2 + +- Per-ecosystem overrides (`cargo-cyclonedx` for Rust, `cyclonedx-gradle-plugin` for + Java) — Phase 4's evaluation decides if the Syft-on-deb baseline here is good enough + first. +- `appliance-build` fetching/merging these sidecars into the per-image document — that's + Phase 3 (CP-13466), which explicitly depends on this phase completing. +- DCT/Hyperscale sidecars — deferred in the top-level design, not filed as a story yet. +- The `devops-gate` publishing switch (CSV → CycloneDX) — unrelated to sidecar generation. + +## Implementation status + +- [x] `SBOM_DEEP_SCAN` classification added to all 40 packages' `config.sh` (§5). +- [x] `sbom-deep-scan` field added to `query-packages.sh` (§2). +- [x] `generate_sbom()` added to `lib/common.sh` and wired as `stage generate_sbom` in + `buildpkg.sh` (§4), including the multi-`.deb` merge case. +- [x] CI lint added (§3), landed in the same change as the classification pass. +- [x] Verified locally: `verify-sbom-scan-flag.sh` and the existing + `verify-query-packages.sh` both pass; `shellcheck`/`shfmt` clean on every touched file. + +## Follow-ups before merging + +1. Validate against a real `syft`/`cyclonedx-cli` build host: confirm `syft scan ` + works as expected (the open question in §4), and run an actual `zfs` build (multi-`.deb` + package) and `masking` build (single-`.deb`, Java+npm) end-to-end — confirm + `$WORKDIR/artifacts/.cdx.json` is produced, schema-valid, and lands in S3. +2. File as sub-tasks of DLPX-98872. \ No newline at end of file diff --git a/lib/common.sh b/lib/common.sh index 1faab9c7..d4ed860c 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1461,6 +1461,67 @@ function store_build_info() { fi } +# +# Generate a per-package CycloneDX SBOM sidecar by running Syft against +# this package's built .deb(s). Only packages that bundle third-party +# composition (jars, npm, wheels, Rust crates, ...) opt in via +# SBOM_DEEP_SCAN="true" in their config.sh -- everything else is left as +# a flat pkg:deb component by appliance-build's base chroot scan, so a +# deb scan here would add nothing. The resulting .cdx.json is +# dropped in $WORKDIR/artifacts/ alongside the .deb(s), where it's picked +# up by the same S3 sync as every other build artifact -- no separate +# upload path needed. +# +function generate_sbom() { + if [[ "$SBOM_DEEP_SCAN" != "true" ]]; then + return 0 + fi + + local debs=("$WORKDIR/artifacts/"*.deb) + if [[ ! -e "${debs[0]}" ]]; then + die "SBOM_DEEP_SCAN is set but no .deb was found in" \ + "'$WORKDIR/artifacts'" + fi + + local sbom_file="$WORKDIR/artifacts/$PACKAGE.cdx.json" + local sbom_scratch_dir + sbom_scratch_dir="$(logmust mktemp -d)" + + # + # A package can emit more than one .deb from a single build (e.g. + # "zfs" splits into zfs-dkms, zfsutils-linux, etc.) -- scan each one + # into its own document, then merge them into a single sidecar so + # there's exactly one .cdx.json per package, matching how + # appliance-build associates a sidecar to a package via COMPONENTS. + # + local deb sbom_parts=() + for deb in "${debs[@]}"; do + local part + part="$sbom_scratch_dir/$(basename "$deb").cdx.json" + logmust syft scan "$deb" \ + --source-name "$PACKAGE" \ + --source-version "$PACKAGE_VERSION" \ + -o "cyclonedx-json@1.6=$part" + sbom_parts+=("$part") + done + + if [[ ${#sbom_parts[@]} -eq 1 ]]; then + logmust cp "${sbom_parts[0]}" "$sbom_file" + else + logmust cyclonedx-cli merge \ + --input-files "${sbom_parts[@]}" \ + --output-format json \ + --output-file "$sbom_file" + fi + logmust rm -rf "$sbom_scratch_dir" + + logmust cyclonedx-cli validate \ + --input-file "$sbom_file" \ + --input-format json \ + --input-version v1_6 \ + --fail-on-errors +} + function set_secret_build_args() { _SECRET_BUILD_ARGS=() diff --git a/packages/bcc/config.sh b/packages/bcc/config.sh index 047b6922..1094b594 100644 --- a/packages/bcc/config.sh +++ b/packages/bcc/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bcc.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL=https://github.com/iovisor/bcc.git UPSTREAM_GIT_BRANCH=master diff --git a/packages/challenge-response/config.sh b/packages/challenge-response/config.sh index 997f96f8..6c29a67d 100644 --- a/packages/challenge-response/config.sh +++ b/packages/challenge-response/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/challenge-response.git" +SBOM_DEEP_SCAN="false" function prepare() { install_build_deps_from_control_file diff --git a/packages/cloud-init/config.sh b/packages/cloud-init/config.sh index dd720656..aab2ad00 100644 --- a/packages/cloud-init/config.sh +++ b/packages/cloud-init/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/cloud-init.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE=cloud-init diff --git a/packages/connstat/config.sh b/packages/connstat/config.sh index 4e50e1b4..08ea16ed 100644 --- a/packages/connstat/config.sh +++ b/packages/connstat/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/connstat.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="@linux-kernel dwarves" function prepare() { diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh index 36b06aca..a53a234e 100644 --- a/packages/containerized-masking/config.sh +++ b/packages/containerized-masking/config.sh @@ -40,6 +40,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" # PACKAGE_NEEDS_DOCKER="true" MEND_SCAN_APPLICABLE="true" +SBOM_DEEP_SCAN="true" MEND_SCAN_IMAGES="'delphix-masking-proxy', 'delphix-masking-database', 'delphix-masking-app'" SKIP_COPYRIGHTS_CHECK=true diff --git a/packages/crash-python/config.sh b/packages/crash-python/config.sh index 495597fc..ac839c98 100644 --- a/packages/crash-python/config.sh +++ b/packages/crash-python/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/crash-python.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust install_build_deps_from_control_file diff --git a/packages/crypt-blowfish/config.sh b/packages/crypt-blowfish/config.sh index 04244a4b..b1f26638 100644 --- a/packages/crypt-blowfish/config.sh +++ b/packages/crypt-blowfish/config.sh @@ -17,6 +17,8 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/crypt-blowfish.git" +SBOM_DEEP_SCAN="false" + SKIP_COPYRIGHTS_CHECK=true function build() { diff --git a/packages/cyclonedx-cli/config.sh b/packages/cyclonedx-cli/config.sh index 3ee39875..ed56ae59 100755 --- a/packages/cyclonedx-cli/config.sh +++ b/packages/cyclonedx-cli/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/cyclonedx-cli.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-go/config.sh b/packages/delphix-go/config.sh index f9926295..9dfc17d1 100755 --- a/packages/delphix-go/config.sh +++ b/packages/delphix-go/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-go.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-kernel/config.sh b/packages/delphix-kernel/config.sh index 125836ff..c556cc25 100644 --- a/packages/delphix-kernel/config.sh +++ b/packages/delphix-kernel/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-kernel.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="@linux-kernel" function prepare() { diff --git a/packages/delphix-platform/config.sh b/packages/delphix-platform/config.sh index 42e4ff4c..c83aa809 100644 --- a/packages/delphix-platform/config.sh +++ b/packages/delphix-platform/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-platform.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust cd "$WORKDIR/repo" diff --git a/packages/delphix-rust/config.sh b/packages/delphix-rust/config.sh index 4f1f06c0..c1613a2b 100755 --- a/packages/delphix-rust/config.sh +++ b/packages/delphix-rust/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-rust.git" +SBOM_DEEP_SCAN="true" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 6d1a9c05..480f79d0 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -19,6 +19,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/saml-app.git" MEND_SCAN_APPLICABLE="true" MEND_SCAN_USING_SUDO="true" +SBOM_DEEP_SCAN="true" function prepare() { logmust install_pkgs openjdk-17-jdk-headless: diff --git a/packages/docker-python-image/config.sh b/packages/docker-python-image/config.sh index b80e6172..47f1f501 100644 --- a/packages/docker-python-image/config.sh +++ b/packages/docker-python-image/config.sh @@ -17,12 +17,14 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/docker-python-image.git" +SBOM_DEEP_SCAN="false" + # # debian/rules' override_dh_install runs 'docker pull' to fetch the python # image it repackages, so the build needs a docker daemon. The build container # gets the host's daemon through its socket rather than running one of its own. # -PACKAGE_NEEDS_DOCKER="true" +PACKAGE_NEEDS_DOCKER="false" function prepare() { logmust install_build_deps_from_control_file diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh index 0387a3a7..3f29a512 100644 --- a/packages/drgn/config.sh +++ b/packages/drgn/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/drgn.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="libkdumpfile" UPSTREAM_GIT_URL="https://github.com/osandov/drgn.git" diff --git a/packages/dwarves/config.sh b/packages/dwarves/config.sh index 46965c54..86cfcc66 100644 --- a/packages/dwarves/config.sh +++ b/packages/dwarves/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dwarves.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL="https://github.com/acmel/dwarves.git" UPSTREAM_GIT_BRANCH="master" diff --git a/packages/fluentd-gems/config.sh b/packages/fluentd-gems/config.sh index b511f1bb..2110330b 100644 --- a/packages/fluentd-gems/config.sh +++ b/packages/fluentd-gems/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/fluentd-gems.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/gdb-python/config.sh b/packages/gdb-python/config.sh index 014215f0..c36d22dd 100644 --- a/packages/gdb-python/config.sh +++ b/packages/gdb-python/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/gdb-python.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="libkdumpfile" function prepare() { diff --git a/packages/grub2/config.sh b/packages/grub2/config.sh index 82452184..e280c484 100644 --- a/packages/grub2/config.sh +++ b/packages/grub2/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none +SBOM_DEEP_SCAN="false" SKIP_COPYRIGHTS_CHECK=true URI="s3://release-de-images/internal-artifacts/2025.3.0.1/1.0.53/input-artifacts/combined-packages/packages/grub2" diff --git a/packages/host-jdks/config.sh b/packages/host-jdks/config.sh index 5aec9a30..d0b93c80 100755 --- a/packages/host-jdks/config.sh +++ b/packages/host-jdks/config.sh @@ -23,6 +23,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/host-jdks.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/libkdumpfile/config.sh b/packages/libkdumpfile/config.sh index d1dd6f52..91976030 100644 --- a/packages/libkdumpfile/config.sh +++ b/packages/libkdumpfile/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/libkdumpfile.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL="https://codeberg.org/ptesarik/libkdumpfile.git" UPSTREAM_GIT_BRANCH="tip" diff --git a/packages/linux-kernel-aws/config.sh b/packages/linux-kernel-aws/config.sh index b2596d2f..1f00b06c 100644 --- a/packages/linux-kernel-aws/config.sh +++ b/packages/linux-kernel-aws/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-azure/config.sh b/packages/linux-kernel-azure/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-azure/config.sh +++ b/packages/linux-kernel-azure/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-gcp/config.sh b/packages/linux-kernel-gcp/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-gcp/config.sh +++ b/packages/linux-kernel-gcp/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-generic/config.sh b/packages/linux-kernel-generic/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-generic/config.sh +++ b/packages/linux-kernel-generic/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-oracle/config.sh b/packages/linux-kernel-oracle/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-oracle/config.sh +++ b/packages/linux-kernel-oracle/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/makedumpfile/config.sh b/packages/makedumpfile/config.sh index 8eb407a5..3ad8172d 100644 --- a/packages/makedumpfile/config.sh +++ b/packages/makedumpfile/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/makedumpfile.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE="makedumpfile" diff --git a/packages/masking/config.sh b/packages/masking/config.sh index 92163bdb..2229e02d 100644 --- a/packages/masking/config.sh +++ b/packages/masking/config.sh @@ -20,6 +20,7 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" MEND_SCAN_APPLICABLE="true" +SBOM_DEEP_SCAN="true" function prepare() { logmust read_list "$WORKDIR/repo/packaging/build-dependencies" diff --git a/packages/misc-debs/config.sh b/packages/misc-debs/config.sh index 19fa7c7d..32bb43bd 100644 --- a/packages/misc-debs/config.sh +++ b/packages/misc-debs/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none +SBOM_DEEP_SCAN="false" SKIP_COPYRIGHTS_CHECK=true # diff --git a/packages/nfs-utils/config.sh b/packages/nfs-utils/config.sh index 15e3c725..639097d6 100644 --- a/packages/nfs-utils/config.sh +++ b/packages/nfs-utils/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/nfs-utils.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE=nfs-utils diff --git a/packages/performance-diagnostics/config.sh b/packages/performance-diagnostics/config.sh index 4733a8c9..6d666c36 100644 --- a/packages/performance-diagnostics/config.sh +++ b/packages/performance-diagnostics/config.sh @@ -18,6 +18,7 @@ # DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/performance-diagnostics.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust install_build_deps_from_control_file diff --git a/packages/ptools/config.sh b/packages/ptools/config.sh index 459cc4ca..54aef875 100644 --- a/packages/ptools/config.sh +++ b/packages/ptools/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/ptools.git" +SBOM_DEEP_SCAN="true" function prepare() { logmust install_pkgs cargo devscripts diff --git a/packages/python-rtslib-fb/config.sh b/packages/python-rtslib-fb/config.sh index 024c025e..99c334ef 100644 --- a/packages/python-rtslib-fb/config.sh +++ b/packages/python-rtslib-fb/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/python-rtslib-fb.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE=python-rtslib-fb diff --git a/packages/savedump/config.sh b/packages/savedump/config.sh index 1e9bdda2..ba291b44 100644 --- a/packages/savedump/config.sh +++ b/packages/savedump/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/savedump.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust install_build_deps_from_control_file diff --git a/packages/sdb/config.sh b/packages/sdb/config.sh index 3b36fed4..119abc18 100644 --- a/packages/sdb/config.sh +++ b/packages/sdb/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/sdb.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL="https://github.com/sdimitro/sdb.git" UPSTREAM_GIT_BRANCH="develop" diff --git a/packages/syft/config.sh b/packages/syft/config.sh index 987ee8f6..d4d40573 100755 --- a/packages/syft/config.sh +++ b/packages/syft/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/syft.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/targetcli-fb/config.sh b/packages/targetcli-fb/config.sh index abdc2ea3..3841b56b 100644 --- a/packages/targetcli-fb/config.sh +++ b/packages/targetcli-fb/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/targetcli-fb.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="python-rtslib-fb" UPSTREAM_SOURCE_PACKAGE=targetcli-fb diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh index e7fe6b3a..171b111a 100644 --- a/packages/virtualization/config.sh +++ b/packages/virtualization/config.sh @@ -27,6 +27,7 @@ PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks" # PACKAGE_NEEDS_DOCKER="true" MEND_SCAN_APPLICABLE="true" +SBOM_DEEP_SCAN="true" function prepare() { logmust read_list "$WORKDIR/repo/appliance/packaging/build-dependencies" diff --git a/packages/windows-connector/config.sh b/packages/windows-connector/config.sh index 40479cd2..185083da 100644 --- a/packages/windows-connector/config.sh +++ b/packages/windows-connector/config.sh @@ -25,6 +25,7 @@ # DEFAULT_PACKAGE_GIT_URL="none" SKIP_COPYRIGHTS_CHECK=true +SBOM_DEEP_SCAN="true" function fetch() { PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index bd1edce5..de13dcb1 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/zfs.git" PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves" +SBOM_DEEP_SCAN="true" UPSTREAM_GIT_URL="https://github.com/openzfs/zfs.git" UPSTREAM_GIT_BRANCH="master" diff --git a/query-packages.sh b/query-packages.sh index d92c7c23..612f70ed 100755 --- a/query-packages.sh +++ b/query-packages.sh @@ -25,7 +25,7 @@ source "$TOP/lib/common.sh" # export LOGGING=false -ALL_OUTPUT_FIELDS=(name git-url dependencies can-update mend-scan mend-scan-images mend-scan-using-sudo) +ALL_OUTPUT_FIELDS=(name git-url dependencies can-update mend-scan mend-scan-images mend-scan-using-sudo sbom-deep-scan) function usage() { local output_fields="${ALL_OUTPUT_FIELDS[*]}" @@ -73,6 +73,7 @@ function print_package() { mend-scan) outarray+=("${MEND_SCAN_APPLICABLE:-none}") ;; mend-scan-images) outarray+=("${MEND_SCAN_IMAGES:-none}") ;; mend-scan-using-sudo) outarray+=("${MEND_SCAN_USING_SUDO:-none}") ;; + sbom-deep-scan) outarray+=("${SBOM_DEEP_SCAN:-none}") ;; dependencies) for dep in $PACKAGE_DEPENDENCIES; do check_package_exists "$dep" From 0b4e66b8a5249e1b48fe7db7caf6fd2732e5adb4 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Wed, 9 Sep 2026 17:17:00 +0530 Subject: [PATCH 2/8] fixed packages/docker-python-image/config.sh --- packages/docker-python-image/config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/docker-python-image/config.sh b/packages/docker-python-image/config.sh index 47f1f501..9379b26a 100644 --- a/packages/docker-python-image/config.sh +++ b/packages/docker-python-image/config.sh @@ -24,7 +24,7 @@ SBOM_DEEP_SCAN="false" # image it repackages, so the build needs a docker daemon. The build container # gets the host's daemon through its socket rather than running one of its own. # -PACKAGE_NEEDS_DOCKER="false" +PACKAGE_NEEDS_DOCKER="true" function prepare() { logmust install_build_deps_from_control_file From 48d68231062c61743ca24b4c41aa0346199dd43c Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Thu, 10 Sep 2026 00:57:24 +0530 Subject: [PATCH 3/8] DLPX-98872 fix: install syft/cyclonedx-cli before generate_sbom, fix version generate_sbom() assumed syft/cyclonedx-cli were already on PATH, but nothing installed them into the linux-pkg build container -- Phase 1 only installs them onto the appliance-build host. Confirmed via a real pre-push run: every SBOM_DEEP_SCAN package failed with "syft: command not found". Fixed by declaring syft/cyclonedx-cli as PACKAGE_DEPENDENCIES on all 8 flagged packages (masking, virtualization, delphix-sso-app, containerized-masking, windows-connector, zfs, ptools, delphix-rust), and having generate_sbom() install them from $DEPDIR before scanning -- the same pattern zfs already uses for delphix-rust. Also fixes --source-version showing up blank for packages (like masking) that don't set $PACKAGE_VERSION themselves: read the version back out of the built .deb via dpkg-deb instead of relying on a shell variable that doesn't reliably survive to this stage. verify-query-packages.sh's zfs dependency-list assertion updated to match the new PACKAGE_DEPENDENCIES. --- .github/scripts/verify-query-packages.sh | 2 +- lib/common.sh | 27 ++++++++++++++++++++++-- packages/containerized-masking/config.sh | 7 ++++++ packages/delphix-rust/config.sh | 7 ++++++ packages/delphix-sso-app/config.sh | 7 ++++++ packages/masking/config.sh | 7 ++++++ packages/ptools/config.sh | 7 ++++++ packages/virtualization/config.sh | 2 +- packages/windows-connector/config.sh | 7 ++++++ packages/zfs/config.sh | 2 +- 10 files changed, 70 insertions(+), 5 deletions(-) diff --git a/.github/scripts/verify-query-packages.sh b/.github/scripts/verify-query-packages.sh index c4051171..b6f8f64b 100755 --- a/.github/scripts/verify-query-packages.sh +++ b/.github/scripts/verify-query-packages.sh @@ -64,7 +64,7 @@ test "$(TARGET_KERNEL_FLAVORS=generic ./query-packages.sh list linux-kernel)" == # Check that when a package has multiple dependencies they are printed in the # expected format. test "$(TARGET_KERNEL_FLAVORS="generic aws" ./query-packages.sh single -o dependencies zfs)" == \ - "linux-kernel-generic,linux-kernel-aws,delphix-rust,delphix-go,dwarves" + "linux-kernel-generic,linux-kernel-aws,delphix-rust,delphix-go,dwarves,syft,cyclonedx-cli" # Check that the output from the appliance list contains zfs and # delphix-platform packages. Note, we explicitly do not use grep -q here as it diff --git a/lib/common.sh b/lib/common.sh index d4ed860c..5b4b4aed 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1483,6 +1483,18 @@ function generate_sbom() { "'$WORKDIR/artifacts'" fi + # + # syft/cyclonedx-cli are build-host-only tooling (never shipped in + # any product package), fetched the same way any other linux-pkg + # build dependency is: declared in PACKAGE_DEPENDENCIES, populated + # into $DEPDIR by the "fetch_dependencies" stage, installed here. + # This mirrors appliance-build's build-ancillary-repository.sh, + # which installs the same two packages onto the appliance-build host + # for the same reason. + # + check_env DEPDIR + logmust install_pkgs "$DEPDIR"/syft/*.deb "$DEPDIR"/cyclonedx-cli/*.deb + local sbom_file="$WORKDIR/artifacts/$PACKAGE.cdx.json" local sbom_scratch_dir sbom_scratch_dir="$(logmust mktemp -d)" @@ -1496,11 +1508,22 @@ function generate_sbom() { # local deb sbom_parts=() for deb in "${debs[@]}"; do - local part + local part deb_version part="$sbom_scratch_dir/$(basename "$deb").cdx.json" + # + # Read the version back out of the .deb itself, rather than + # relying on $PACKAGE_VERSION: by this point in the build, + # $PACKAGE_VERSION may no longer hold the final, + # revision-suffixed version set_changelog() wrote into the + # package (e.g. it's empty for packages that don't set it + # explicitly themselves, unlike syft/cyclonedx-cli's own + # config.sh). dpkg-deb reads the actual, authoritative + # version of the artifact being scanned. + # + deb_version="$(dpkg-deb -f "$deb" Version)" logmust syft scan "$deb" \ --source-name "$PACKAGE" \ - --source-version "$PACKAGE_VERSION" \ + --source-version "$deb_version" \ -o "cyclonedx-json@1.6=$part" sbom_parts+=("$part") done diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh index a53a234e..ccff8400 100644 --- a/packages/containerized-masking/config.sh +++ b/packages/containerized-masking/config.sh @@ -41,6 +41,13 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" PACKAGE_NEEDS_DOCKER="true" MEND_SCAN_APPLICABLE="true" SBOM_DEEP_SCAN="true" + +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" MEND_SCAN_IMAGES="'delphix-masking-proxy', 'delphix-masking-database', 'delphix-masking-app'" SKIP_COPYRIGHTS_CHECK=true diff --git a/packages/delphix-rust/config.sh b/packages/delphix-rust/config.sh index c1613a2b..581eada4 100755 --- a/packages/delphix-rust/config.sh +++ b/packages/delphix-rust/config.sh @@ -19,6 +19,13 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-rust.git" SBOM_DEEP_SCAN="true" +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" + function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 480f79d0..175f3d61 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -21,6 +21,13 @@ MEND_SCAN_APPLICABLE="true" MEND_SCAN_USING_SUDO="true" SBOM_DEEP_SCAN="true" +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" + function prepare() { logmust install_pkgs openjdk-17-jdk-headless: } diff --git a/packages/masking/config.sh b/packages/masking/config.sh index 2229e02d..2c42f4f9 100644 --- a/packages/masking/config.sh +++ b/packages/masking/config.sh @@ -22,6 +22,13 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" MEND_SCAN_APPLICABLE="true" SBOM_DEEP_SCAN="true" +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" + function prepare() { logmust read_list "$WORKDIR/repo/packaging/build-dependencies" logmust install_pkgs "${_RET_LIST[@]}" diff --git a/packages/ptools/config.sh b/packages/ptools/config.sh index 54aef875..e6fefae1 100644 --- a/packages/ptools/config.sh +++ b/packages/ptools/config.sh @@ -19,6 +19,13 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/ptools.git" SBOM_DEEP_SCAN="true" +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" + function prepare() { logmust install_pkgs cargo devscripts } diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh index 171b111a..f960df04 100644 --- a/packages/virtualization/config.sh +++ b/packages/virtualization/config.sh @@ -19,7 +19,7 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" -PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks" +PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks syft cyclonedx-cli" # # The build packages a docker image (see '-Ddockerize=true' below), which needs # a docker daemon. The build container gets the host's daemon through its diff --git a/packages/windows-connector/config.sh b/packages/windows-connector/config.sh index 185083da..4b96c987 100644 --- a/packages/windows-connector/config.sh +++ b/packages/windows-connector/config.sh @@ -27,6 +27,13 @@ DEFAULT_PACKAGE_GIT_URL="none" SKIP_COPYRIGHTS_CHECK=true SBOM_DEEP_SCAN="true" +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" + function fetch() { PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" logmust fetch_repo_from_git diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index de13dcb1..36187eb0 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -17,7 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/zfs.git" -PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves" +PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves syft cyclonedx-cli" SBOM_DEEP_SCAN="true" UPSTREAM_GIT_URL="https://github.com/openzfs/zfs.git" From bdabf2bbdf0da2e32e7040c1b6ffc3482e0fa37d Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Thu, 10 Sep 2026 15:53:32 +0530 Subject: [PATCH 4/8] DLPX-98872: suppress Syft's file-metadata component in per-package SBOM generate_sbom() didn't set SYFT_FILE_METADATA_SELECTION=none, so every sidecar carried an extra CycloneDX "file" component for the .deb itself (SHA-1/SHA-256 hashes plus the absolute build-workspace path) alongside the real pkg:deb component -- confirmed in delphix-sso-app.cdx.json from a real pre-push build. Same fix appliance-build's 95-generate-sbom.binary hook already applies, for the same reason: a flat, per-package pkg:deb document shouldn't carry per-file noise or leak the build machine's local path. --- lib/common.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/common.sh b/lib/common.sh index 5b4b4aed..ae5fdf2e 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1521,7 +1521,14 @@ function generate_sbom() { # version of the artifact being scanned. # deb_version="$(dpkg-deb -f "$deb" Version)" - logmust syft scan "$deb" \ + # + # SYFT_FILE_METADATA_SELECTION=none suppresses Syft's default + # per-file "file" component (with SHA-1/SHA-256 hashes and the + # absolute build-workspace path baked in) -- noise that doesn't + # belong in a per-package pkg:deb sidecar. Same reasoning as + # appliance-build's 95-generate-sbom.binary hook. + # + SYFT_FILE_METADATA_SELECTION=none logmust syft scan "$deb" \ --source-name "$PACKAGE" \ --source-version "$deb_version" \ -o "cyclonedx-json@1.6=$part" From 703a4de7d2978acedc11112bf54d2403b4bf1574 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Thu, 10 Sep 2026 17:27:14 +0530 Subject: [PATCH 5/8] DLPX-98872: pin cyclonedx-cli merge's --output-version to v1_6 Multi-.deb packages (delphix-rust: delphix-rust + delphix-rust-src; zfs similarly) go through the cyclonedx-cli merge path in generate_sbom(), which had no --output-version pinned. cyclonedx-cli merge defaults to the newest spec version it supports (1.7), not the 1.6 Syft emitted, so the merged document then failed the very next validate step: Incorrect schema version: expected 1.6 actual 1.7 Confirmed via a real pre-push delphix-rust build. Single-.deb packages never hit this, since the merge step is skipped entirely for them (a plain cp). Co-Authored-By: Claude Sonnet 5 --- lib/common.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/common.sh b/lib/common.sh index ae5fdf2e..844c3e40 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1538,9 +1538,18 @@ function generate_sbom() { if [[ ${#sbom_parts[@]} -eq 1 ]]; then logmust cp "${sbom_parts[0]}" "$sbom_file" else + # + # --output-version must be pinned explicitly: cyclonedx-cli + # merge defaults to the newest spec version it supports (1.7), + # not the 1.6 that Syft emitted and that the validate call + # below (and every other producer in this pipeline) targets -- + # left unset, the merged doc fails validation with "Incorrect + # schema version: expected 1.6 actual 1.7". + # logmust cyclonedx-cli merge \ --input-files "${sbom_parts[@]}" \ --output-format json \ + --output-version v1_6 \ --output-file "$sbom_file" fi logmust rm -rf "$sbom_scratch_dir" From 8923f539e3aa2fd2a9fe904d0ca3caaec6d6e669 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Thu, 10 Sep 2026 18:52:16 +0530 Subject: [PATCH 6/8] DLPX-98872: strict 1:1 .deb <-> BOM mapping, drop cyclonedx-cli merge Reviewer feedback: each .deb should have its own distinct BOM file with a common filename prefix, not one merged SBOM per source package. The prior approach scanned each .deb separately then merged them via cyclonedx-cli merge into a single .cdx.json -- correct per the top-level design doc's stated intent ("one package-level SBOM... associated with all of that package's debs"), but that assumption didn't survive review. generate_sbom() now writes one .deb.cdx.json per .deb, independently scanned and validated, with no merge step at all. This also removes the cyclonedx-cli merge --output-version bug entirely, since there's no merge left to have a version mismatch in. Also reconfirms compliance with the reviewer's other point: syft, cyclonedx-cli, and all linux-kernel-* packages are SBOM_DEEP_SCAN=false (unaffected by this change, verified separately) -- no SBOM is generated for build-host tooling or 3rd-party kernel forks. Design doc updated to match: architecture diagram, the "multiple .debs per package" resolution, and the implementation-status/follow-ups sections now describe the 1:1 mapping and the real-build testing already done, instead of the merge approach and its since-superseded open question about cyclonedx-cli's --output-version default. Co-Authored-By: Claude Sonnet 5 --- ...6-09-08-sbom-per-package-sidecar-design.md | 140 +++++++++++------- lib/common.sh | 76 ++++------ 2 files changed, 110 insertions(+), 106 deletions(-) diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md index 7e97bccc..eb32f98a 100644 --- a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -115,41 +115,29 @@ function generate_sbom() { "'$WORKDIR/artifacts'" fi - local sbom_file="$WORKDIR/artifacts/$PACKAGE.cdx.json" - local sbom_scratch_dir - sbom_scratch_dir="$(logmust mktemp -d)" - - # A package can emit more than one .deb from a single build (e.g. - # "zfs" splits into zfs-dkms, zfsutils-linux, etc.) -- scan each one - # into its own document, then merge into a single sidecar so there's - # exactly one .cdx.json per package, matching how - # appliance-build associates a sidecar to a package via COMPONENTS. - local deb sbom_parts=() + check_env DEPDIR + logmust install_pkgs "$DEPDIR"/syft/*.deb "$DEPDIR"/cyclonedx-cli/*.deb + + # One sidecar per .deb, not per package: a package that emits more + # than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux, + # etc.) gets one .deb.cdx.json per .deb -- a strict + # 1:1 mapping, no merging across a package's .deb(s). + local deb for deb in "${debs[@]}"; do - local part - part="$sbom_scratch_dir/$(basename "$deb").cdx.json" - logmust syft scan "$deb" \ + local sbom_file deb_version + sbom_file="$WORKDIR/artifacts/$(basename "$deb").cdx.json" + deb_version="$(dpkg-deb -f "$deb" Version)" + SYFT_FILE_METADATA_SELECTION=none logmust syft scan "$deb" \ --source-name "$PACKAGE" \ - --source-version "$PACKAGE_VERSION" \ - -o "cyclonedx-json@1.6=$part" - sbom_parts+=("$part") + --source-version "$deb_version" \ + -o "cyclonedx-json@1.6=$sbom_file" + + logmust cyclonedx-cli validate \ + --input-file "$sbom_file" \ + --input-format json \ + --input-version v1_6 \ + --fail-on-errors done - - if [[ ${#sbom_parts[@]} -eq 1 ]]; then - logmust cp "${sbom_parts[0]}" "$sbom_file" - else - logmust cyclonedx-cli merge \ - --input-files "${sbom_parts[@]}" \ - --output-format json \ - --output-file "$sbom_file" - fi - logmust rm -rf "$sbom_scratch_dir" - - logmust cyclonedx-cli validate \ - --input-file "$sbom_file" \ - --input-format json \ - --input-version v1_6 \ - --fail-on-errors } ``` @@ -170,15 +158,26 @@ flag, means no package needs its own override for the baseline case (only per-ec overrides, out of scope for Phase 2, would override the function in a package's `config.sh`, the same way packages already override `build()`). -**Resolved — multiple `.deb`s per package:** handled above via per-deb scan + `cyclonedx-cli -merge` into one sidecar. Verify against a real `zfs` build before merging (see *Follow-ups*). - -**Open — Syft's `.deb`-scan support:** not yet verified against the real `syft`/ -`cyclonedx-cli` binaries in this environment (neither is installed here). Confirm the pinned -`delphix-syft` version (`SYFT_VERSION` in the `syft` packaging repo) supports scanning a -standalone `.deb` file as a source (Phase 1 only exercises Syft against a directory/rootfs, -not a single `.deb`). If unsupported, fall back to extracting the `.deb`'s `data.tar.*` into -a temp dir first and scanning that directory instead. +**Resolved — multiple `.deb`s per package:** a reviewer flagged that the original +merge-into-one-sidecar approach (below, kept here for history) doesn't give a clean 1:1 +mapping between a `.deb` and its BOM. Changed to: one `.deb.cdx.json` per +`.deb`, no merge, no `cyclonedx-cli merge` step at all — each `.deb`'s sidecar is fully +independent and filename-matched to it. This is a real divergence from the top-level +design doc (CP-13456), which called for "one package-level SBOM... associated with all +of that package's debs via `COMPONENTS`" — that assumption didn't survive review. Phase 3 +(`appliance-build`'s consumer, not yet built) will need to associate each `.deb` with its +own sidecar directly by filename, not go through a package-level indirection. + +*(For reference, the approach this replaced: scan each `.deb` into its own document, then +`cyclonedx-cli merge --output-version v1_6` them into a single `.cdx.json`. Two +real bugs were found and fixed while that was still in place, both still relevant to the +current code since they're not specific to the merge step: `syft`/`cyclonedx-cli` need to +be installed from `$DEPDIR` before use — see §4's `install_pkgs` line — and Syft's default +file-metadata component needs suppressing via `SYFT_FILE_METADATA_SELECTION=none`.)* + +**Resolved — Syft's `.deb`-scan support:** confirmed working against the real `syft`/ +`cyclonedx-cli` binaries via an actual pre-push build (`delphix-sso-app`, `delphix-rust`) — +`syft scan ` scans a standalone `.deb` file directly, no extraction needed. ### 5. Package classification @@ -225,10 +224,11 @@ per-package artifact directories at all. | stage build --> $WORKDIR/artifacts/*.deb | | stage store_build_info --> GIT_HASH, BUILD_INFO, ... | | stage generate_sbom --> [only if SBOM_DEEP_SCAN="true"] | -| syft scan -o cyclonedx-json@1.6 | -| (merge via cyclonedx-cli if >1 .deb) | -| --> $WORKDIR/artifacts/.cdx.json | -| cyclonedx-cli validate --fail-on-errors | +| for each .deb: | +| syft scan -o cyclonedx-json@1.6 | +| --> .deb.cdx.json | +| cyclonedx-cli validate --fail-on-errors| +| (strict 1:1 .deb <-> BOM, no merging) | | stage post_build_checks | +------------------------------------+--------------------------------------+ | @@ -238,14 +238,14 @@ per-package artifact directories at all. v +---------------------------------------------------------------------------+ | S3: combined-packages/packages// | -| *.deb | -| .cdx.json <-- NEW: sidecar, sits beside the deb(s) | +| .deb | +| .deb.cdx.json <-- NEW: one per .deb, same prefix | | GIT_HASH, BUILD_INFO, ... (unchanged) | +---------------------------------------------------------------------------+ | | (Phase 3, not in scope here: - | appliance-build fetches + merges - | this sidecar into the Phase 1 base) + | appliance-build fetches each .deb's + | own sidecar by filename match) v [out of scope for Phase 2] ``` @@ -265,15 +265,41 @@ per-package artifact directories at all. - [x] `SBOM_DEEP_SCAN` classification added to all 40 packages' `config.sh` (§5). - [x] `sbom-deep-scan` field added to `query-packages.sh` (§2). - [x] `generate_sbom()` added to `lib/common.sh` and wired as `stage generate_sbom` in - `buildpkg.sh` (§4), including the multi-`.deb` merge case. + `buildpkg.sh` (§4), producing a strict 1:1 `.deb` → `.cdx.json` mapping (§4). - [x] CI lint added (§3), landed in the same change as the classification pass. - [x] Verified locally: `verify-sbom-scan-flag.sh` and the existing `verify-query-packages.sh` both pass; `shellcheck`/`shfmt` clean on every touched file. - -## Follow-ups before merging - -1. Validate against a real `syft`/`cyclonedx-cli` build host: confirm `syft scan ` - works as expected (the open question in §4), and run an actual `zfs` build (multi-`.deb` - package) and `masking` build (single-`.deb`, Java+npm) end-to-end — confirm - `$WORKDIR/artifacts/.cdx.json` is produced, schema-valid, and lands in S3. +- [x] Verified against a real build host: `delphix-sso-app` (single-`.deb`) and + `delphix-rust` (multi-`.deb`) pre-push builds both produce valid, schema-checked + CycloneDX 1.6 sidecars in S3, correctly named per `.deb`. +- [x] Confirmed compliant with reviewer feedback: `syft`, `cyclonedx-cli`, and all + `linux-kernel-*` packages are `SBOM_DEEP_SCAN="false"` (no SBOM generated for + build-host tooling or 3rd-party kernel forks), and the `.deb` ↔ `.cdx.json` mapping + is now strictly 1:1 with a shared filename prefix. + +## Bugs found and fixed during real-build testing + +Not caught by CI — only surfaced by actually running builds: + +1. **`syft`/`cyclonedx-cli` command not found.** Nothing installed them into the + `linux-pkg` build container (Phase 1 only solved this for the `appliance-build` host). + Fixed by declaring them as `PACKAGE_DEPENDENCIES` on all 8 flagged packages and + installing from `$DEPDIR` in `generate_sbom()`. +2. **Blank `--source-version`.** `$PACKAGE_VERSION` doesn't reliably survive to the + `generate_sbom` stage for packages that don't set it themselves (unlike + `syft`/`cyclonedx-cli`'s own `config.sh`). Fixed by reading the version back out of the + built `.deb` via `dpkg-deb -f "$deb" Version`. +3. **Stray Syft "file" component**, carrying file hashes and an absolute build-workspace + path, polluting the sidecar. Fixed with `SYFT_FILE_METADATA_SELECTION=none`, matching + `appliance-build`'s `95-generate-sbom.binary` hook. +4. **`cyclonedx-cli merge` defaulted to spec version 1.7**, failing the subsequent + `--input-version v1_6` validate call. Moot now that merging was removed entirely per + the 1:1-mapping change above, but the same lesson applies to any future + `cyclonedx-cli` invocation: pin `--output-version` explicitly, don't rely on its + default. + +## Follow-ups + +1. Phase 3 (`appliance-build`'s consumer) needs to key off the 1:1 `.deb` ↔ `.cdx.json` + filename mapping established here, not a package-level `COMPONENTS` indirection. 2. File as sub-tasks of DLPX-98872. \ No newline at end of file diff --git a/lib/common.sh b/lib/common.sh index 844c3e40..0877d6e6 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1462,15 +1462,16 @@ function store_build_info() { } # -# Generate a per-package CycloneDX SBOM sidecar by running Syft against -# this package's built .deb(s). Only packages that bundle third-party -# composition (jars, npm, wheels, Rust crates, ...) opt in via -# SBOM_DEEP_SCAN="true" in their config.sh -- everything else is left as -# a flat pkg:deb component by appliance-build's base chroot scan, so a -# deb scan here would add nothing. The resulting .cdx.json is -# dropped in $WORKDIR/artifacts/ alongside the .deb(s), where it's picked -# up by the same S3 sync as every other build artifact -- no separate -# upload path needed. +# Generate a CycloneDX SBOM sidecar for each of this package's built +# .deb(s) by running Syft against it. Only packages that bundle +# third-party composition (jars, npm, wheels, Rust crates, ...) opt in +# via SBOM_DEEP_SCAN="true" in their config.sh -- everything else is +# left as a flat pkg:deb component by appliance-build's base chroot +# scan, so a deb scan here would add nothing. Each .cdx.json +# is dropped in $WORKDIR/artifacts/ alongside the .deb it describes -- +# a strict 1:1 mapping, no merging across a package's .deb(s) -- where +# it's picked up by the same S3 sync as every other build artifact, no +# separate upload path needed. # function generate_sbom() { if [[ "$SBOM_DEEP_SCAN" != "true" ]]; then @@ -1495,21 +1496,18 @@ function generate_sbom() { check_env DEPDIR logmust install_pkgs "$DEPDIR"/syft/*.deb "$DEPDIR"/cyclonedx-cli/*.deb - local sbom_file="$WORKDIR/artifacts/$PACKAGE.cdx.json" - local sbom_scratch_dir - sbom_scratch_dir="$(logmust mktemp -d)" - # - # A package can emit more than one .deb from a single build (e.g. - # "zfs" splits into zfs-dkms, zfsutils-linux, etc.) -- scan each one - # into its own document, then merge them into a single sidecar so - # there's exactly one .cdx.json per package, matching how - # appliance-build associates a sidecar to a package via COMPONENTS. + # One sidecar per .deb, not per package: a package that emits more + # than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux, + # etc.) gets one .deb.cdx.json per .deb, each a + # standalone document scoped to that .deb alone. No merging across + # .debs -- keeps a strict 1:1 mapping between a .deb and its BOM, + # with the .deb's own filename as the common prefix. # - local deb sbom_parts=() + local deb for deb in "${debs[@]}"; do - local part deb_version - part="$sbom_scratch_dir/$(basename "$deb").cdx.json" + local sbom_file deb_version + sbom_file="$WORKDIR/artifacts/$(basename "$deb").cdx.json" # # Read the version back out of the .deb itself, rather than # relying on $PACKAGE_VERSION: by this point in the build, @@ -1525,40 +1523,20 @@ function generate_sbom() { # SYFT_FILE_METADATA_SELECTION=none suppresses Syft's default # per-file "file" component (with SHA-1/SHA-256 hashes and the # absolute build-workspace path baked in) -- noise that doesn't - # belong in a per-package pkg:deb sidecar. Same reasoning as + # belong in a per-deb sidecar. Same reasoning as # appliance-build's 95-generate-sbom.binary hook. # SYFT_FILE_METADATA_SELECTION=none logmust syft scan "$deb" \ --source-name "$PACKAGE" \ --source-version "$deb_version" \ - -o "cyclonedx-json@1.6=$part" - sbom_parts+=("$part") - done - - if [[ ${#sbom_parts[@]} -eq 1 ]]; then - logmust cp "${sbom_parts[0]}" "$sbom_file" - else - # - # --output-version must be pinned explicitly: cyclonedx-cli - # merge defaults to the newest spec version it supports (1.7), - # not the 1.6 that Syft emitted and that the validate call - # below (and every other producer in this pipeline) targets -- - # left unset, the merged doc fails validation with "Incorrect - # schema version: expected 1.6 actual 1.7". - # - logmust cyclonedx-cli merge \ - --input-files "${sbom_parts[@]}" \ - --output-format json \ - --output-version v1_6 \ - --output-file "$sbom_file" - fi - logmust rm -rf "$sbom_scratch_dir" + -o "cyclonedx-json@1.6=$sbom_file" - logmust cyclonedx-cli validate \ - --input-file "$sbom_file" \ - --input-format json \ - --input-version v1_6 \ - --fail-on-errors + logmust cyclonedx-cli validate \ + --input-file "$sbom_file" \ + --input-format json \ + --input-version v1_6 \ + --fail-on-errors + done } function set_secret_build_args() { From 3d769d77915ea679a3777e052d142c1585ce8419 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Fri, 11 Sep 2026 15:59:57 +0530 Subject: [PATCH 7/8] DLPX-98872: provision syft/cyclonedx-cli as generic build tooling Review feedback: generate_sbom() is a default hook, inherited unmodified by every SBOM_DEEP_SCAN package, so the tooling it runs is the hook's concern -- a package's config.sh should not have to declare it. Those tools belong with the rest of linux-pkg's generic build dependencies, installed prior to any package build. All 8 flagged packages therefore drop PACKAGE_DEPENDENCIES="syft cyclonedx-cli" (virtualization and zfs are restored to their original dependency lists), and setup.sh gains install_sbom_tools() alongside install_awscli/install_shfmt. Unlike everything else installed there, delphix-syft/delphix-cyclonedx-cli are Delphix-built .debs with no apt source -- the container's apt sources are only the Ubuntu primary mirror and the PPA secondary mirror -- so they are fetched from the same S3 location fetch_dependencies() uses and installed by path. apt rather than dpkg, since delphix-cyclonedx-cli has real libicu dependencies. The install is best-effort: setup.sh is package-agnostic, so it cannot skip itself when the package being built is syft or cyclonedx-cli. A hard failure on a missing artifact would break every build on a branch where neither has been published yet, including their own, so a missing artifact warns and continues. generate_sbom() checks for the tools itself and fails loudly, affecting only the builds that need them. Trade-off worth noting: with no PACKAGE_DEPENDENCIES entry, Jenkins's static dependency graph no longer knows these packages relate to syft/cyclonedx-cli, so build-order batching and the rebuild-dependents- on-syft-change cascade no longer apply to them. Documented in the spec, along with the rejected alternative of deriving the dependency from SBOM_DEEP_SCAN in load_package_config(). verify-query-packages.sh's zfs assertion is restored accordingly. Co-Authored-By: Claude Opus 5 --- .github/scripts/verify-query-packages.sh | 2 +- ...6-09-08-sbom-per-package-sidecar-design.md | 57 ++++++++++-- lib/common.sh | 86 ++++++++++++++++--- packages/containerized-masking/config.sh | 6 -- packages/delphix-rust/config.sh | 7 -- packages/delphix-sso-app/config.sh | 7 -- packages/masking/config.sh | 7 -- packages/ptools/config.sh | 7 -- packages/virtualization/config.sh | 2 +- packages/windows-connector/config.sh | 7 -- packages/zfs/config.sh | 2 +- setup.sh | 6 ++ 12 files changed, 136 insertions(+), 60 deletions(-) diff --git a/.github/scripts/verify-query-packages.sh b/.github/scripts/verify-query-packages.sh index b6f8f64b..c4051171 100755 --- a/.github/scripts/verify-query-packages.sh +++ b/.github/scripts/verify-query-packages.sh @@ -64,7 +64,7 @@ test "$(TARGET_KERNEL_FLAVORS=generic ./query-packages.sh list linux-kernel)" == # Check that when a package has multiple dependencies they are printed in the # expected format. test "$(TARGET_KERNEL_FLAVORS="generic aws" ./query-packages.sh single -o dependencies zfs)" == \ - "linux-kernel-generic,linux-kernel-aws,delphix-rust,delphix-go,dwarves,syft,cyclonedx-cli" + "linux-kernel-generic,linux-kernel-aws,delphix-rust,delphix-go,dwarves" # Check that the output from the appliance list contains zfs and # delphix-platform packages. Note, we explicitly do not use grep -q here as it diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md index eb32f98a..2538ff65 100644 --- a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -115,8 +115,14 @@ function generate_sbom() { "'$WORKDIR/artifacts'" fi - check_env DEPDIR - logmust install_pkgs "$DEPDIR"/syft/*.deb "$DEPDIR"/cyclonedx-cli/*.deb + # syft/cyclonedx-cli come from setup.sh's install_sbom_tools(), not + # from anything this package declares (see below). That install is + # best-effort, so check here rather than letting "command not + # found" surface mid-scan. + local tool + for tool in syft cyclonedx-cli; do + command -v "$tool" >/dev/null || die "'$tool' is not installed..." + done # One sidecar per .deb, not per package: a package that emits more # than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux, @@ -171,8 +177,8 @@ own sidecar directly by filename, not go through a package-level indirection. *(For reference, the approach this replaced: scan each `.deb` into its own document, then `cyclonedx-cli merge --output-version v1_6` them into a single `.cdx.json`. Two real bugs were found and fixed while that was still in place, both still relevant to the -current code since they're not specific to the merge step: `syft`/`cyclonedx-cli` need to -be installed from `$DEPDIR` before use — see §4's `install_pkgs` line — and Syft's default +current code since they're not specific to the merge step: `syft`/`cyclonedx-cli` have to +actually be installed in the build container before use — see §6 — and Syft's default file-metadata component needs suppressing via `SYFT_FILE_METADATA_SELECTION=none`.)* **Resolved — Syft's `.deb`-scan support:** confirmed working against the real `syft`/ @@ -203,6 +209,44 @@ the top-level design's Tooling section separately calls out a possible Go overri binary-build-info cataloger that may already cover it) — revisit under Phase 4's evaluation if gaps are found. +### 6. Provisioning syft/cyclonedx-cli — generic build tooling, not a package dependency + +`generate_sbom()` is a **default hook**: defined once in `lib/common.sh` and inherited +unmodified by all 8 flagged packages (none of them override it, unlike e.g. `zfs`'s own +`build()`). The tooling it runs is therefore the hook's concern, not its callers' — so no +package's `config.sh` declares `syft`/`cyclonedx-cli` anywhere. They are installed with the +rest of the generic build tooling in `setup.sh`, which runs before every package build: + +```bash +logmust install_awscli +logmust install_sbom_tools # must follow install_awscli -- fetches from S3 +logmust install_shfmt +``` + +`install_sbom_tools()` (`lib/common.sh`) is modelled on the existing `install_awscli()`/ +`install_shfmt()` non-apt installers, with one difference: `delphix-syft`/ +`delphix-cyclonedx-cli` are Delphix-built `.deb`s with no apt source (the container's apt +sources are only the Ubuntu primary mirror plus the PPA secondary mirror), so they're +fetched from the same S3 location `fetch_dependencies()` uses — +`get_package_dependency_s3_url()` → `aws s3 cp --recursive` → `apt-get install ` +(apt, not dpkg, because `delphix-cyclonedx-cli` has real `libicu` dependencies). + +**Best-effort by design.** `setup.sh` is package-agnostic — it has no idea which package is +about to be built — so it cannot skip itself when the package being built *is* `syft` or +`cyclonedx-cli`. Hard-failing on a missing artifact would therefore break *every* build on +a branch where those two haven't been published yet, including their own. Instead a missing +artifact warns and continues, and `generate_sbom()` checks for the tools itself and fails +loudly — so only the builds that actually need them are affected. + +**Rejected alternative:** deriving `PACKAGE_DEPENDENCIES += "syft cyclonedx-cli"` from +`SBOM_DEEP_SCAN` in `load_package_config()`. That also keeps it out of each `config.sh`, +is scoped to just the 8 packages, and keeps the relationship visible to Jenkins's static +dependency graph (so `syft` batches before its dependents and a `syft` rebuild cascades to +them). It was implemented and working, but sits in the per-package dependency layer rather +than the generic installed-prior layer, which is not what review asked for. Noting the +trade-off explicitly: with the `setup.sh` approach, Jenkins no longer knows these packages +relate to `syft`/`cyclonedx-cli`, so that batching/rebuild-cascade behaviour is lost. + ### S3 upload — no new plumbing needed Confirmed by reading `devops-gate/jenkins/jobs/pipelines/linux_pkg_build_package.groovy`'s @@ -283,8 +327,9 @@ Not caught by CI — only surfaced by actually running builds: 1. **`syft`/`cyclonedx-cli` command not found.** Nothing installed them into the `linux-pkg` build container (Phase 1 only solved this for the `appliance-build` host). - Fixed by declaring them as `PACKAGE_DEPENDENCIES` on all 8 flagged packages and - installing from `$DEPDIR` in `generate_sbom()`. + Fixed by installing them from `setup.sh` with the rest of the generic build tooling + (§6). An interim fix declared them as `PACKAGE_DEPENDENCIES` on each flagged package + instead; that was replaced per review feedback. 2. **Blank `--source-version`.** `$PACKAGE_VERSION` doesn't reliably survive to the `generate_sbom` stage for packages that don't set it themselves (unlike `syft`/`cyclonedx-cli`'s own `config.sh`). Fixed by reading the version back out of the diff --git a/lib/common.sh b/lib/common.sh index 0877d6e6..4d65ab99 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -619,6 +619,66 @@ function install_shfmt() { echo "shfmt version $(shfmt -version) is installed." } +# +# Install the tooling generate_sbom() needs. That is a default hook, shared +# unmodified by every package that sets SBOM_DEEP_SCAN, so the tools it runs +# belong with the rest of the generic build tooling installed here rather than +# being declared as a dependency by each of those packages individually. +# +# Unlike everything else installed from setup.sh these are Delphix-built +# packages with no apt source, so they are fetched from the same S3 location +# fetch_dependencies() pulls a package's dependencies from, then installed by +# path. apt rather than dpkg, because delphix-cyclonedx-cli has real +# dependencies (libicu and the usual shared libraries) that dpkg will not +# resolve. +# +# Best-effort by design: this runs before *every* package build, including the +# builds of syft and cyclonedx-cli themselves, and on a branch where neither +# has been published yet there is nothing to fetch. Failing hard would break +# every build on such a branch rather than just SBOM generation, so a missing +# artifact warns and moves on; generate_sbom() checks for the tools itself and +# fails loudly, for the only builds that actually need them. +# +function install_sbom_tools() { + local pkg s3url tmpdir + local debs=() + + tmpdir="$(mktemp -d)" || die "Failed to create a temporary directory" + + for pkg in syft cyclonedx-cli; do + # + # Run in a command substitution so that a failure to resolve + # the URL (get_package_dependency_s3_url dies when a package + # has no published artifacts) leaves $s3url empty here instead + # of aborting setup. + # + s3url="$( + get_package_dependency_s3_url "$pkg" >/dev/null 2>&1 + echo "$_RET" + )" + if [[ -z "$s3url" ]]; then + echo "WARNING: no published artifacts found for '$pkg';" \ + "skipping the SBOM tooling install. Builds of" \ + "packages that set SBOM_DEEP_SCAN will fail until" \ + "'$pkg' has been built for this branch." + logmust rm -rf "$tmpdir" + return 0 + fi + + [[ "$s3url" != */ ]] && s3url="$s3url/" + logmust mkdir -p "$tmpdir/$pkg" + logmust aws s3 cp --only-show-errors --recursive \ + "$s3url" "$tmpdir/$pkg/" + done + + debs=("$tmpdir"/*/*.deb) + [[ -e "${debs[0]}" ]] || + die "No .deb found in the fetched syft/cyclonedx-cli artifacts" + + logmust install_pkgs "${debs[@]}" + logmust rm -rf "$tmpdir" +} + # # Install kernel headers packages for all target kernels. # The kernel packages are fetched from S3. @@ -1485,16 +1545,22 @@ function generate_sbom() { fi # - # syft/cyclonedx-cli are build-host-only tooling (never shipped in - # any product package), fetched the same way any other linux-pkg - # build dependency is: declared in PACKAGE_DEPENDENCIES, populated - # into $DEPDIR by the "fetch_dependencies" stage, installed here. - # This mirrors appliance-build's build-ancillary-repository.sh, - # which installs the same two packages onto the appliance-build host - # for the same reason. - # - check_env DEPDIR - logmust install_pkgs "$DEPDIR"/syft/*.deb "$DEPDIR"/cyclonedx-cli/*.deb + # syft/cyclonedx-cli are part of the generic build tooling installed + # by setup.sh (install_sbom_tools()) before any package is built, + # rather than something each SBOM_DEEP_SCAN package declares for + # itself -- this is a default hook, so what it needs is its own + # concern, not its callers'. That install is best-effort, so check + # here rather than letting "command not found" surface from the + # middle of a scan. + # + local tool + for tool in syft cyclonedx-cli; do + command -v "$tool" >/dev/null || + die "'$tool' is not installed, so no SBOM can be" \ + "generated for '$PACKAGE'. It is provisioned by" \ + "install_sbom_tools() in setup.sh; check that" \ + "run's output for why it was skipped." + done # # One sidecar per .deb, not per package: a package that emits more diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh index ccff8400..d1cff120 100644 --- a/packages/containerized-masking/config.sh +++ b/packages/containerized-masking/config.sh @@ -42,12 +42,6 @@ PACKAGE_NEEDS_DOCKER="true" MEND_SCAN_APPLICABLE="true" SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" MEND_SCAN_IMAGES="'delphix-masking-proxy', 'delphix-masking-database', 'delphix-masking-app'" SKIP_COPYRIGHTS_CHECK=true diff --git a/packages/delphix-rust/config.sh b/packages/delphix-rust/config.sh index 581eada4..c1613a2b 100755 --- a/packages/delphix-rust/config.sh +++ b/packages/delphix-rust/config.sh @@ -19,13 +19,6 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-rust.git" SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" - function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 175f3d61..480f79d0 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -21,13 +21,6 @@ MEND_SCAN_APPLICABLE="true" MEND_SCAN_USING_SUDO="true" SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" - function prepare() { logmust install_pkgs openjdk-17-jdk-headless: } diff --git a/packages/masking/config.sh b/packages/masking/config.sh index 2c42f4f9..2229e02d 100644 --- a/packages/masking/config.sh +++ b/packages/masking/config.sh @@ -22,13 +22,6 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" MEND_SCAN_APPLICABLE="true" SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" - function prepare() { logmust read_list "$WORKDIR/repo/packaging/build-dependencies" logmust install_pkgs "${_RET_LIST[@]}" diff --git a/packages/ptools/config.sh b/packages/ptools/config.sh index e6fefae1..54aef875 100644 --- a/packages/ptools/config.sh +++ b/packages/ptools/config.sh @@ -19,13 +19,6 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/ptools.git" SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" - function prepare() { logmust install_pkgs cargo devscripts } diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh index f960df04..171b111a 100644 --- a/packages/virtualization/config.sh +++ b/packages/virtualization/config.sh @@ -19,7 +19,7 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" -PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks syft cyclonedx-cli" +PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks" # # The build packages a docker image (see '-Ddockerize=true' below), which needs # a docker daemon. The build container gets the host's daemon through its diff --git a/packages/windows-connector/config.sh b/packages/windows-connector/config.sh index 4b96c987..185083da 100644 --- a/packages/windows-connector/config.sh +++ b/packages/windows-connector/config.sh @@ -27,13 +27,6 @@ DEFAULT_PACKAGE_GIT_URL="none" SKIP_COPYRIGHTS_CHECK=true SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" - function fetch() { PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" logmust fetch_repo_from_git diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index 36187eb0..de13dcb1 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -17,7 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/zfs.git" -PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves syft cyclonedx-cli" +PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves" SBOM_DEEP_SCAN="true" UPSTREAM_GIT_URL="https://github.com/openzfs/zfs.git" diff --git a/setup.sh b/setup.sh index cb8248b5..c0a00815 100755 --- a/setup.sh +++ b/setup.sh @@ -263,6 +263,12 @@ function install_awscli() { logmust install_awscli +# +# Must follow install_awscli: the syft/cyclonedx-cli packages it installs are +# fetched from S3, so the aws CLI has to be present first. +# +logmust install_sbom_tools + logmust install_shfmt logmust git config --global user.email "eng@delphix.com" From 7934898c7f4e7b1ec9f93e340a49867726d4a79d Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Fri, 11 Sep 2026 16:00:05 +0530 Subject: [PATCH 8/8] DLPX-98872: explain the SBOM_DEEP_SCAN decision in the lint failure Review feedback: the lint told you that a package was unclassified but not how to classify it, leaving whoever adds the next package to go read lib/common.sh or the spec to find out. State the rule where it is actually needed -- 1st-party packages set "true" so the third-party components they package internally land in the product's aggregate SBOM; 3rd-party forks of Debian packages and anything not shipped in a product set "false", since appliance-build's image-level scan already covers those as a flat pkg:deb component. Co-Authored-By: Claude Opus 5 --- .github/scripts/verify-sbom-scan-flag.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/scripts/verify-sbom-scan-flag.sh b/.github/scripts/verify-sbom-scan-flag.sh index 94691ddd..7b97e587 100755 --- a/.github/scripts/verify-sbom-scan-flag.sh +++ b/.github/scripts/verify-sbom-scan-flag.sh @@ -18,6 +18,16 @@ if [[ -n "$unclassified" ]]; then echo "The following packages have not set SBOM_DEEP_SCAN (\"true\" or" \ "\"false\") in their config.sh:" echo "$unclassified" + echo + echo "Set it to \"true\" for 1st-party packages, so that the" \ + "third-party components they package internally (jars, npm" \ + "modules, Rust crates, ...) are included in the product's" \ + "aggregate SBOM." + echo "Set it to \"false\" for 3rd-party forks of Debian packages, and" \ + "for packages that are not included in a shipping product:" \ + "those are already covered as a flat pkg:deb component by" \ + "appliance-build's image-level scan, so a deep scan here would" \ + "add nothing." exit 1 fi