From 707b536ab2ba81107c9f053562d6047c122ec00b Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 01/71] Join LOCALVERSION with '+' so release candidates sort below finals systemd-boot sorts boot entries on the BLS 'version' key, which kernel-install writes verbatim from uname -r. Under systemd's version comparison: 7.2.0-rc7-qcom-next-20260821 > 7.2.0-qcom-next-20260826 so every release candidate outranked the final release that followed it, and an rc daily stayed the default boot entry after the release was installed. The deciding comparison is a plain strcmp of "rc" against "qcom", where 'r' > 'q'; there is no notion of a pre-release involved. debian/rules already rewrites -rcN to ~rcN, but only for the Debian version, because a package name may not contain '~' and the binary packages are named linux-image-. So dpkg ordered these correctly all along and only the boot menu was wrong. Joining the suffix with '+' fixes it: systemd compares the separator before the chunk behind it, and '-' < '+'. This is not a local invention -- it is why Debian's own kernels sort correctly (linux-image-7.1.10+deb14-amd64). Upstream bindeb-pkg has the identical ~rc/-rc split and gets away with it for a different reason: it puts a numeric commit count in that slot, and systemd ranks digits above letters. qcom-next was the only scheme putting a lowercase word there. The Debian version is unchanged, byte for byte: DATE_PART reads the date off the end of LOCALVERSION, which still matches. Only uname -r and the binary package names change. Note this renames every binary package. Already-installed kernels under the old names are not upgraded or removed by it -- they keep their existing sort position until purged, so the fix applies to newly installed kernels only. Signed-off-by: Christopher Obbard --- ci/scripts/derive-localversion.sh | 28 +++++++++++++++++++++------- debian/rules | 8 ++++---- prepare-source.sh | 12 ++++++------ 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/ci/scripts/derive-localversion.sh b/ci/scripts/derive-localversion.sh index d36e9b0b..90108396 100755 --- a/ci/scripts/derive-localversion.sh +++ b/ci/scripts/derive-localversion.sh @@ -6,14 +6,28 @@ set -euo pipefail # Derive the LOCALVERSION suffix from a kernel variant and resolved ref. # # For dated tag builds (ref ends in -YYYYMMDD): -# Produces --. -# Example: qcom-next-7.2-rc3-20260722 -> -qcom-next-20260722 +# Produces +-. +# Example: qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722 # # For branch-tip builds (ref does not end in a date): # Uses the kernel variant and a short SHA for uniqueness. -# Example: qcom-next @ 07f50dc44edd -> -qcom-next-g07f50dc44edd +# Example: qcom-next @ 07f50dc44edd -> +qcom-next-g07f50dc44edd # --sha is required for branch-tip builds. # +# Why the leading '+' and not '-': +# The suffix ends up in KERNELRELEASE (uname -r), which is the 'version' field +# systemd-boot sorts BLS entries on. systemd compares the separator before the +# chunk behind it, and '-' < '+', so joining with '+' puts every -rcN release +# candidate BELOW the final release that follows it: +# +# 7.2.0-rc7+qcom-next-20260821 < 7.2.0+qcom-next-20260826 +# +# Joining with '-' instead falls through to a plain strcmp of "rc" against +# "qcom", where 'r' > 'q', and every rc outranks its own final release in the +# boot menu. This is the same trick Debian's own kernels rely on +# (linux-image-7.1.10+deb14-amd64). It does not affect the Debian version, +# which spells the release candidate ~rcN and orders correctly either way. +# # Usage: # ci/scripts/derive-localversion.sh --variant qcom-next --ref qcom-next-7.2-rc3-20260722 # ci/scripts/derive-localversion.sh --variant arduino --ref main --sha 07f50dc44edd @@ -24,8 +38,8 @@ set -euo pipefail # --sha SHA Short commit SHA (required for branch-tip builds). # # Output: -# LOCALVERSION suffix printed to stdout (e.g. -qcom-next-20260722). -# Always starts with a dash. +# LOCALVERSION suffix printed to stdout (e.g. +qcom-next-20260722). +# Always starts with a plus. # # Exit codes: # 0 Success. @@ -60,7 +74,7 @@ done # the variant supplies the stable package identity used in LOCALVERSION. if [[ "$REF" =~ -([0-9]{8})$ ]]; then DATE="${BASH_REMATCH[1]}" - echo "-${VARIANT}-${DATE}" + echo "+${VARIANT}-${DATE}" else # Branch-tip build: need SHA for uniqueness. [[ -n "$SHA" ]] || { @@ -69,5 +83,5 @@ else } # Use first 12 chars of SHA for a compact but unambiguous suffix. SHORT_SHA="${SHA:0:12}" - echo "-${VARIANT}-g${SHORT_SHA}" + echo "+${VARIANT}-g${SHORT_SHA}" fi diff --git a/debian/rules b/debian/rules index 1de36c97..3cdb4378 100755 --- a/debian/rules +++ b/debian/rules @@ -47,7 +47,7 @@ KREL_FILE := debian/kernel.release # debian/control before invoking any debian/rules targets. # # Inputs (make variables): -# LOCALVERSION Suffix appended to the base kernel version, e.g. -qcom-next-20260722. +# LOCALVERSION Suffix appended to the base kernel version, e.g. +qcom-next-20260722. # Required unless KVER is given explicitly. # KVER Full kernelrelease string (uname -r). Overrides LOCALVERSION. # KVER_EXTRA Optional extra suffix appended after LOCALVERSION, e.g. -ci42. @@ -70,7 +70,7 @@ KREL_FILE := debian/kernel.release # so an empty DKMS_MODULES is a genuine "bundle nothing" # rather than the missing-manifest misconfiguration that # the bundler rejects. -# debian/localversion Plain file containing the LOCALVERSION suffix (e.g. -qcom-next-20260722). +# debian/localversion Plain file containing the LOCALVERSION suffix (e.g. +qcom-next-20260722). # Read by override_dh_auto_build to pass LOCALVERSION= to make. # debian/pkgversion Plain file containing the full Debian package version # (e.g. 7.2.0-rc3+20260722-0qcom1). @@ -93,8 +93,8 @@ prepare: else \ [ -n "$(LOCALVERSION)" ] || { \ echo "ERROR: LOCALVERSION is required when KVER is not set."; \ - echo " Pass LOCALVERSION= (e.g. LOCALVERSION=-qcom-next-20260722)"; \ - echo " or KVER= (e.g. KVER=7.2.0-rc3-qcom-next-20260722)"; \ + echo " Pass LOCALVERSION= (e.g. LOCALVERSION=+qcom-next-20260722)"; \ + echo " or KVER= (e.g. KVER=7.2.0-rc3+qcom-next-20260722)"; \ exit 1; \ }; \ echo "Reading base kernel version from kernel Makefile..."; \ diff --git a/prepare-source.sh b/prepare-source.sh index f0cbbc72..b5db6efe 100755 --- a/prepare-source.sh +++ b/prepare-source.sh @@ -52,7 +52,7 @@ OPTIONS: -d, --distro DISTRO Target suite: trixie|forky|sid|noble|questing|resolute (default: $DEFAULT_DISTRO) --localversion SUFFIX LOCALVERSION suffix appended to the base kernel - version (e.g. -qcom-next-20260722). + version (e.g. +qcom-next-20260722). Auto-detected from git tag if not specified. --kver-extra SUFFIX Extra suffix appended to the final KVER (e.g. -ci42). @@ -101,7 +101,7 @@ EXAMPLES: # Full CI invocation with all options $0 --source-dir /path/to/kernel \\ --distro trixie \\ - --localversion -qcom-next-20260722 \\ + --localversion +qcom-next-20260722 \\ --srcpkg linux-qcom-next \\ --binpkg linux-image-qcom-next \\ --debian-revision 0qcom1 \\ @@ -153,13 +153,13 @@ VALID_DISTROS=(noble questing resolute trixie forky sid unstable) [[ -d "$DEBIAN_DIR" ]] || { log_error "Debian dir not found: $DEBIAN_DIR"; exit 1; } # ── Helper: derive LOCALVERSION from a tag name ────────────────────────────── -# qcom-next-7.2-rc3-20260722 -> -qcom-next-20260722 +# qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722 _auto_localversion() { local tag="$1" if [[ "$tag" =~ ^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]+)$ ]]; then - echo "-${BASH_REMATCH[1]}-${BASH_REMATCH[2]}" + echo "+${BASH_REMATCH[1]}-${BASH_REMATCH[2]}" else - echo "-$tag" + echo "+$tag" fi } @@ -172,7 +172,7 @@ if [[ -z "$LOCALVERSION" ]]; then else log_warn "LOCALVERSION not set and no exact git tag found." log_warn "Package will be named linux-image- (no branch/date suffix)." - log_warn "Use --localversion to specify, e.g.: --localversion -qcom-next-20260722" + log_warn "Use --localversion to specify, e.g.: --localversion +qcom-next-20260722" fi fi From 0ebfcd09cdda6a50177278ee41f00ac1566545ae Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 02/71] Give the Debian revision stub a packaging revision digit The stub was a bare '0qli', which left a packaging-only rebuild -- same kernel snapshot, changed debian/ -- with no field to bump. Neither of the other two components can absorb it: suite_suffix_mapping is a per-suite constant shared by every row, and the delivery suffix is the Daily/Release marker. Bumping either to force a rebuild through would corrupt what they mean. Make the trailing digit part of the stub, so a rebuild is 0qli1 -> 0qli2 and lands uniformly across every suite: 0qli1~bpo13+1~ < 0qli1~bpo13+1 < 0qli2~bpo13+1 derive-debian-revision.sh now rejects a stub that does not end in a digit, alongside the existing "must not end in ~" check. Both guard the same thing: a stub that quietly loses a field it is supposed to carry. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-deb.yml | 6 +++--- README.md | 12 ++++++------ ci/build-matrix.json | 8 ++++---- ci/scripts/derive-debian-revision.sh | 17 ++++++++++++++--- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml index 9da42580..aca93182 100644 --- a/.github/workflows/build-kernel-deb.yml +++ b/.github/workflows/build-kernel-deb.yml @@ -53,7 +53,7 @@ on: debian-version-stub: description: 'Advanced: Debian version stub; the selected suite''s mapped suffix and a Daily-style trailing ~ are applied automatically' required: false - default: '0qli' + default: '0qli1' localversion: description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' required: false @@ -124,7 +124,7 @@ on: debian-revision: description: 'Debian revision component of the package version' type: string - default: '0qli~' + default: '0qli1~' localversion: description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' type: string @@ -301,7 +301,7 @@ jobs: # non-promoting. env: DEBIAN_REVISION_INPUT: ${{ inputs.debian-revision }} - DEBIAN_VERSION_STUB_INPUT: ${{ inputs.debian-version-stub || '0qli' }} + DEBIAN_VERSION_STUB_INPUT: ${{ inputs.debian-version-stub || '0qli1' }} run: | set -euo pipefail INPUT_REVISION="$DEBIAN_REVISION_INPUT" diff --git a/README.md b/README.md index 1c60c355..ebf73381 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ The final Production matrix is conceptually: "binpkg": "linux-image-qcom-next", "kernel_config": [], "dkms": ["kgsl", "camx", "iris-vpu"], - "debian_version_stub": "0qli", + "debian_version_stub": "0qli1", "debian_version_suffix": "~" }, { @@ -76,7 +76,7 @@ The final Production matrix is conceptually: "binpkg": "linux-image-qcom-next", "kernel_config": [], "dkms": ["kgsl", "camx", "iris-vpu"], - "debian_version_stub": "0qli", + "debian_version_stub": "0qli1", "debian_version_suffix": "", "target_workspace": "qli" } @@ -93,9 +93,9 @@ above: | Suite | Daily | Release | | --- | --- | --- | -| Trixie | `0qli~bpo13+1~` | `0qli~bpo13+1` | -| Forky | `0qli~` | `0qli` | -| Resolute | `0qli~26.04.1~` | (not a configured Release suite) | +| Trixie | `0qli1~bpo13+1~` | `0qli1~bpo13+1` | +| Forky | `0qli1~` | `0qli1` | +| Resolute | `0qli1~26.04.1~` | (not a configured Release suite) | `~` always sorts below the same prefix without it in Debian version ordering, so Daily always sorts below Release for the same suite and stub. @@ -439,7 +439,7 @@ The available inputs are: | `srcpkg` | `linux-qcom-next` | Advanced source package identity override. | | `binpkg` | `linux-image-qcom-next` | Advanced image metapackage identity override. | | `kernel-config` | Empty | Advanced extra fragments applied on top of all of `debian/config-available/`, e.g. `intree:arch/arm64/configs/qcom_debug.config`. | -| `debian-version-stub` | `0qli` | Advanced Debian version stub. The selected suite's mapped suffix and a Daily-style trailing `~` are applied automatically; direct builds always use Daily semantics since they are build-only and non-promoting. | +| `debian-version-stub` | `0qli1` | Advanced Debian version stub. The selected suite's mapped suffix and a Daily-style trailing `~` are applied automatically; direct builds always use Daily semantics since they are build-only and non-promoting. | | `localversion` | Auto-derived | Advanced explicit `LOCALVERSION` override. | | `kver-extra` | Empty | Advanced kernel-release suffix. | | `debug-build` | `false` | Advanced debug configuration toggle. | diff --git a/ci/build-matrix.json b/ci/build-matrix.json index 655aefcb..a28175d4 100644 --- a/ci/build-matrix.json +++ b/ci/build-matrix.json @@ -26,7 +26,7 @@ "iris-vpu", "audioreach" ], - "debian_version_stub": "0qli", + "debian_version_stub": "0qli1", "debian_version_suffix": "~" }, { @@ -49,7 +49,7 @@ "iris-vpu", "audioreach" ], - "debian_version_stub": "0qli", + "debian_version_stub": "0qli1", "debian_version_suffix": "" }, { @@ -75,7 +75,7 @@ "iris-vpu", "audioreach" ], - "debian_version_stub": "0qli", + "debian_version_stub": "0qli1", "debian_version_suffix": "~" }, { @@ -101,7 +101,7 @@ "iris-vpu", "audioreach" ], - "debian_version_stub": "0qli", + "debian_version_stub": "0qli1", "debian_version_suffix": "" } ] diff --git a/ci/scripts/derive-debian-revision.sh b/ci/scripts/derive-debian-revision.sh index 71b3af8a..008e993e 100755 --- a/ci/scripts/derive-debian-revision.sh +++ b/ci/scripts/derive-debian-revision.sh @@ -15,11 +15,13 @@ set -euo pipefail # context), so the derivation and its validation live in exactly one place. # # Usage: -# ci/scripts/derive-debian-revision.sh --stub 0qli --suite trixie --delivery-type Daily -# ci/scripts/derive-debian-revision.sh --stub 0qli --suite forky --delivery-type Release --matrix-file ci/build-matrix.json +# ci/scripts/derive-debian-revision.sh --stub 0qli1 --suite trixie --delivery-type Daily +# ci/scripts/derive-debian-revision.sh --stub 0qli1 --suite forky --delivery-type Release --matrix-file ci/build-matrix.json # # Options: -# --stub STUB Debian version stub. Must be non-empty and must not +# --stub STUB Debian version stub, e.g. 0qli1. Must end in a digit: +# that digit is the packaging revision, bumped for a +# rebuild of an unchanged kernel snapshot. Must not # end in ~ (the delivery suffix supplies any # trailing ~). Required. # --suite SUITE Target suite; must have an entry in @@ -62,6 +64,15 @@ done [[ -n "$SUITE" ]] || { echo "ERROR: --suite is required" >&2; exit 1; } [[ -n "$DELIVERY_TYPE" ]] || { echo "ERROR: --delivery-type is required" >&2; exit 1; } [[ "$STUB" != *"~" ]] || { echo "ERROR: --stub must not end in ~ (got '$STUB')" >&2; exit 1; } +# The trailing digit is the packaging revision: it is the only field left to +# bump when the kernel snapshot is unchanged but the packaging is rebuilt. +# suite_suffix_mapping is a per-suite constant and cannot carry it, and the +# delivery suffix is the Daily/Release marker, so a stub without a digit leaves +# a rebuild with nowhere to go. +[[ "$STUB" =~ [0-9]$ ]] || { + echo "ERROR: --stub must end in a digit, the packaging revision (got '$STUB'; use '${STUB}1')" >&2 + exit 1 +} [[ -f "$MATRIX_FILE" ]] || { echo "ERROR: Matrix file not found: $MATRIX_FILE" >&2; exit 1; } jq empty "$MATRIX_FILE" 2>/dev/null \ From e7136871b8a70086051c4f419b9ae242db4c77d7 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 03/71] Compose the Debian snapshot forward instead of parsing LOCALVERSION debian/rules recovered the snapshot date by matching the tail of LOCALVERSION, a string it did not build and cannot safely parse. The variant name in the middle is caller-supplied and unbounded, and a branch-tip suffix ends in a 12-hex SHA that can be eight digits: $ echo '+qcom-next-g123456789012' | grep -oE '[0-9]{8}$' 56789012 Such a build was versioned 7.2.0~rc7+56789012-0qli1, naming a snapshot date that never existed and sorting it against real ones. Roughly one branch-tip SHA in ten ends that way, so this is a live bug, not a theoretical one. Anchoring the match would have fixed those refs while leaving the guess in place. Carry the field instead. derive-localversion.sh already knows the snapshot -- it matched the tag to build LOCALVERSION in the first place -- so it now emits both: LOCALVERSION=+qcom-next-20260722 SNAPSHOT=20260722 KEY=VALUE because the workflow appends them straight to GITHUB_ENV. prepare-source.sh gains --snapshot and forwards it, and its tag auto-detect derives both fields in one match. debian/rules takes SNAPSHOT as an input and validates its shape rather than discovering it. Neither caller reconstructs the field when it is absent. An explicit --localversion replaces the derived suffix precisely because the ref no longer describes the build, so parsing it back would reinstate the guess this commit removes. Both paths warn that the Debian version will carry no dated component, which is visible and correctable, unlike a date invented from a SHA. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-deb.yml | 15 ++++++++--- README.md | 2 +- ci/scripts/derive-localversion.sh | 29 ++++++++++++++++----- debian/rules | 27 +++++++++++++------- prepare-source.sh | 35 ++++++++++++++++++++------ 5 files changed, 80 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml index aca93182..c947b8cf 100644 --- a/.github/workflows/build-kernel-deb.yml +++ b/.github/workflows/build-kernel-deb.yml @@ -273,7 +273,7 @@ jobs: echo "KERNEL_SHA=$KERNEL_SHA" >> "$GITHUB_ENV" echo "Kernel HEAD SHA: $KERNEL_SHA" - - name: Derive LOCALVERSION + - name: Derive version fields env: LOCALVERSION_INPUT: ${{ inputs.localversion }} KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} @@ -281,15 +281,21 @@ jobs: set -euo pipefail OVERRIDE_LV="$LOCALVERSION_INPUT" if [[ -n "$OVERRIDE_LV" ]]; then + # An override supplies no snapshot, and the ref it was chosen to + # replace cannot be trusted to describe it. The Debian version then + # carries no dated component; prepare-source.sh warns about the same. echo "Using explicit LOCALVERSION override: $OVERRIDE_LV" echo "LOCALVERSION=$OVERRIDE_LV" >> "$GITHUB_ENV" + echo "SNAPSHOT=" >> "$GITHUB_ENV" else - LOCALVERSION=$(ci/scripts/derive-localversion.sh \ + # Emits LOCALVERSION= and SNAPSHOT= lines, already in GITHUB_ENV form. + FIELDS=$(ci/scripts/derive-localversion.sh \ --variant "$KERNEL_VARIANT_INPUT" \ --ref "$KERNEL_REF" \ --sha "$(echo "$KERNEL_SHA" | cut -c1-12)") - echo "LOCALVERSION=$LOCALVERSION" >> "$GITHUB_ENV" - echo "Derived LOCALVERSION: $LOCALVERSION" + echo "$FIELDS" >> "$GITHUB_ENV" + echo "Derived version fields:" + echo "$FIELDS" fi - name: Resolve Debian revision @@ -402,6 +408,7 @@ jobs: ) [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") + [[ -n "$SNAPSHOT" ]] && ARGS+=(--snapshot "$SNAPSHOT") [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") # Always passed, unlike the optional inputs above: an empty list is a diff --git a/README.md b/README.md index ebf73381..bcb15a68 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ Supporting scripts keep workflow YAML small and testable: | --- | --- | | `ci/scripts/resolve-matrix.sh` | Validates and flattens matrix rows. | | `ci/scripts/resolve-kernel-ref.sh` | Resolves a matrix-selected dated tag or validates a direct ref. | -| `ci/scripts/derive-localversion.sh` | Derives `LOCALVERSION` from the variant and resolved kernel ref. | +| `ci/scripts/derive-localversion.sh` | Derives the version fields from the variant and resolved kernel ref, printing `LOCALVERSION=` and `SNAPSHOT=` lines. `SNAPSHOT` is the dated component of the Debian version and is empty for branch-tip builds. | | `ci/scripts/derive-debian-revision.sh` | Derives the final suite-specific `debian_revision` from `debian_version_stub`, `suite_suffix_mapping`, and delivery type. | ## Architecture diff --git a/ci/scripts/derive-localversion.sh b/ci/scripts/derive-localversion.sh index 90108396..dea72cd7 100755 --- a/ci/scripts/derive-localversion.sh +++ b/ci/scripts/derive-localversion.sh @@ -3,7 +3,14 @@ # SPDX-License-Identifier: BSD-3-Clause-Clear set -euo pipefail -# Derive the LOCALVERSION suffix from a kernel variant and resolved ref. +# Derive the version fields for a build from a kernel variant and resolved ref. +# +# Emits LOCALVERSION (the kernel release suffix) and SNAPSHOT (the dated +# component of the Debian version), both derived from the ref in one place. +# SNAPSHOT is emitted alongside rather than recovered from LOCALVERSION later: +# reading it back out means guessing where the date ends in a string that also +# carries a variant name and, for branch-tip builds, a hex SHA that can end in +# eight digits of its own. # # For dated tag builds (ref ends in -YYYYMMDD): # Produces +-. @@ -38,8 +45,14 @@ set -euo pipefail # --sha SHA Short commit SHA (required for branch-tip builds). # # Output: -# LOCALVERSION suffix printed to stdout (e.g. +qcom-next-20260722). -# Always starts with a plus. +# Two KEY=VALUE lines on stdout, in GITHUB_ENV / 'set -a' form: +# +# LOCALVERSION=+qcom-next-20260722 +# SNAPSHOT=20260722 +# +# LOCALVERSION always starts with a plus. SNAPSHOT is empty for branch-tip +# builds, which have no date; the Debian version then carries no snapshot +# component at all. # # Exit codes: # 0 Success. @@ -73,8 +86,8 @@ done # Dated tags use a trailing YYYYMMDD snapshot. The matrix selects the tag set; # the variant supplies the stable package identity used in LOCALVERSION. if [[ "$REF" =~ -([0-9]{8})$ ]]; then - DATE="${BASH_REMATCH[1]}" - echo "+${VARIANT}-${DATE}" + SNAPSHOT="${BASH_REMATCH[1]}" + LOCALVERSION="+${VARIANT}-${SNAPSHOT}" else # Branch-tip build: need SHA for uniqueness. [[ -n "$SHA" ]] || { @@ -83,5 +96,9 @@ else } # Use first 12 chars of SHA for a compact but unambiguous suffix. SHORT_SHA="${SHA:0:12}" - echo "+${VARIANT}-g${SHORT_SHA}" + LOCALVERSION="+${VARIANT}-g${SHORT_SHA}" + SNAPSHOT="" fi + +echo "LOCALVERSION=${LOCALVERSION}" +echo "SNAPSHOT=${SNAPSHOT}" diff --git a/debian/rules b/debian/rules index 3cdb4378..cc20c1b9 100755 --- a/debian/rules +++ b/debian/rules @@ -51,6 +51,8 @@ KREL_FILE := debian/kernel.release # Required unless KVER is given explicitly. # KVER Full kernelrelease string (uname -r). Overrides LOCALVERSION. # KVER_EXTRA Optional extra suffix appended after LOCALVERSION, e.g. -ci42. +# SNAPSHOT Dated component of the Debian version, e.g. 20260722. +# Empty for branch-tip builds, which have no snapshot date. # DISTRO Target Debian/Ubuntu suite (default: unstable). # SRCPKG Source package name (default: linux-qcom-next). # BINPKG Binary metapackage name (default: linux-image-qcom-next). @@ -77,6 +79,7 @@ KREL_FILE := debian/kernel.release # Read by override_dh_gencontrol. # --------------------------------------------------------------------------- KVER_EXTRA ?= +SNAPSHOT ?= DISTRO ?= unstable SRCPKG ?= linux-qcom-next BINPKG ?= linux-image-qcom-next @@ -112,11 +115,11 @@ prepare: echo "Appended KVER_EXTRA=$(KVER_EXTRA) -> KVER=$$KVER_RESOLVED"; \ fi; \ \ - # ── Derive package version from KVER ───────────────────────────────────── - # Package version format: +- - # where base_kver = kernel version without the LOCALVERSION suffix, - # and date = the 8-digit date component of LOCALVERSION (if present). - # For branch-tip builds (no date in LOCALVERSION), version = -. + # ── Derive package version ─────────────────────────────────────────────── + # Package version format: +- + # where base_kver = kernel version without the LOCALVERSION suffix, and + # snapshot = the SNAPSHOT input. + # For branch-tip builds, which have no snapshot, version = -. # # Extract base_kver: read directly from kernel Makefile (same as above but # without LOCALVERSION appended), so it is always the pure kernel version. @@ -131,10 +134,16 @@ prepare: EXTRA_DEB=$$(echo "$$EXTRA" | sed 's/^-rc/~rc/'); \ BASE_KVER_CLEAN="$${VER}.$${PATCH}.$${SUB}$${EXTRA_DEB}$${TREE_LOCALVER}"; \ \ - # Extract date from LOCALVERSION: last dash-delimited component if 8 digits. - DATE_PART=$$(echo "$(LOCALVERSION)" | grep -oE '[0-9]{8}$$' || true); \ - if [ -n "$$DATE_PART" ]; then \ - UPSTREAM_VER="$${BASE_KVER_CLEAN}+$${DATE_PART}"; \ + # SNAPSHOT arrives as its own input rather than being recovered from the end + # of LOCALVERSION, because that string is not safely parseable: it carries a + # variant name whose shape is caller-supplied, and for branch-tip builds a + # 12-hex SHA that can itself end in eight digits. + if [ -n "$(SNAPSHOT)" ]; then \ + echo "$(SNAPSHOT)" | grep -qE '^[0-9]{8}$$' || { \ + echo "ERROR: SNAPSHOT must be YYYYMMDD (got '$(SNAPSHOT)')"; \ + exit 1; \ + }; \ + UPSTREAM_VER="$${BASE_KVER_CLEAN}+$(SNAPSHOT)"; \ else \ UPSTREAM_VER="$${BASE_KVER_CLEAN}"; \ fi; \ diff --git a/prepare-source.sh b/prepare-source.sh index b5db6efe..01cfc838 100755 --- a/prepare-source.sh +++ b/prepare-source.sh @@ -54,6 +54,10 @@ OPTIONS: --localversion SUFFIX LOCALVERSION suffix appended to the base kernel version (e.g. +qcom-next-20260722). Auto-detected from git tag if not specified. + --snapshot SNAPSHOT Dated component of the Debian version, e.g. + 20260722. Auto-detected from git tag alongside + --localversion; pass it explicitly whenever + --localversion is passed explicitly. --kver-extra SUFFIX Extra suffix appended to the final KVER (e.g. -ci42). @@ -115,6 +119,7 @@ EOF SOURCE_DIR="" DISTRO="$DEFAULT_DISTRO" LOCALVERSION="" +SNAPSHOT="" KVER_EXTRA="" SRCPKG="$DEFAULT_SRCPKG" BINPKG="$DEFAULT_BINPKG" @@ -127,6 +132,7 @@ while [[ $# -gt 0 ]]; do -s|--source-dir) SOURCE_DIR="$2"; shift 2 ;; -d|--distro) DISTRO="$2"; shift 2 ;; --localversion) LOCALVERSION="$2"; shift 2 ;; + --snapshot) SNAPSHOT="$2"; shift 2 ;; --kver-extra) KVER_EXTRA="$2"; shift 2 ;; --srcpkg) SRCPKG="$2"; shift 2 ;; --binpkg) BINPKG="$2"; shift 2 ;; @@ -152,28 +158,39 @@ VALID_DISTROS=(noble questing resolute trixie forky sid unstable) [[ -d "$DEBIAN_DIR" ]] || { log_error "Debian dir not found: $DEBIAN_DIR"; exit 1; } -# ── Helper: derive LOCALVERSION from a tag name ────────────────────────────── -# qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722 -_auto_localversion() { +# ── Helper: derive LOCALVERSION and SNAPSHOT from a tag name ───────────────── +# qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722 / 20260722 +# +# Both fields come out of the tag together. Recovering the snapshot from +# LOCALVERSION afterwards would mean parsing a string that also holds a variant +# name and, for a branch tip, a hex SHA that can end in eight digits. +_auto_version_fields() { local tag="$1" if [[ "$tag" =~ ^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]+)$ ]]; then - echo "+${BASH_REMATCH[1]}-${BASH_REMATCH[2]}" + LOCALVERSION="+${BASH_REMATCH[1]}-${BASH_REMATCH[2]}" + SNAPSHOT="${BASH_REMATCH[2]}" else - echo "+$tag" + LOCALVERSION="+$tag" + SNAPSHOT="" fi } -# ── Auto-detect LOCALVERSION from git tag (if not provided) ────────────────── +# ── Auto-detect LOCALVERSION and SNAPSHOT from git tag (if not provided) ───── if [[ -z "$LOCALVERSION" ]]; then GIT_TAG=$(git -C "$SOURCE_DIR" describe --tags --exact-match 2>/dev/null || true) if [[ -n "$GIT_TAG" ]]; then - LOCALVERSION="$(_auto_localversion "$GIT_TAG")" - log_info "Auto-detected LOCALVERSION='$LOCALVERSION' from tag '$GIT_TAG'" + _auto_version_fields "$GIT_TAG" + log_info "Auto-detected LOCALVERSION='$LOCALVERSION' SNAPSHOT='$SNAPSHOT' from tag '$GIT_TAG'" else log_warn "LOCALVERSION not set and no exact git tag found." log_warn "Package will be named linux-image- (no branch/date suffix)." log_warn "Use --localversion to specify, e.g.: --localversion +qcom-next-20260722" fi +elif [[ -z "$SNAPSHOT" ]]; then + # An explicit --localversion is not parsed for a snapshot; say so rather + # than silently dropping the dated component from the Debian version. + log_warn "--localversion given without --snapshot: the Debian version will" + log_warn "carry no dated component. Pass --snapshot to supply one." fi log_step "Configuration:" @@ -183,6 +200,7 @@ log_info " Source package: $SRCPKG" log_info " Binary metapkg: $BINPKG" log_info " Debian revision: $DEBIAN_REVISION" [[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" +[[ -n "$SNAPSHOT" ]] && log_info " SNAPSHOT: $SNAPSHOT" [[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" [[ -n "$KERNEL_CONFIG" ]] && log_info " Kernel config: $KERNEL_CONFIG" [[ -n "$DKMS_MODULES" ]] && log_info " DKMS modules: $DKMS_MODULES" @@ -296,6 +314,7 @@ fi log_step "Running debian/rules prepare..." PREPARE_ARGS="DISTRO=$DISTRO SRCPKG=$SRCPKG BINPKG=$BINPKG DEBIAN_REVISION=$DEBIAN_REVISION" [[ -n "$LOCALVERSION" ]] && PREPARE_ARGS="$PREPARE_ARGS LOCALVERSION=$LOCALVERSION" +[[ -n "$SNAPSHOT" ]] && PREPARE_ARGS="$PREPARE_ARGS SNAPSHOT=$SNAPSHOT" [[ -n "$KVER_EXTRA" ]] && PREPARE_ARGS="$PREPARE_ARGS KVER_EXTRA=$KVER_EXTRA" # Spaces are stripped so a list written as "kgsl, camx" stays a single make # argument; debian/rules validates the names it is given. From 8b3f757af55ebdaa50ef73b2eef8685b1b1f37bf Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 04/71] Carry a tag's respin ordinal through the version fields A second tag cut on the same day (qcom-next-7.2-20260826.1) collapsed onto the same version as the first, because every derivation matched only a bare YYYYMMDD. derive-localversion.sh dropped the ordinal; prepare-source.sh matched trailing digits with -([0-9]+)$, which a ".1" suffix cannot satisfy at all, so the tag fell through to the + fallback. Match the date width explicitly with an optional ordinal after it, so the component stays whole: qcom-next-7.2-20260826 -> +qcom-next-20260826 / 20260826 qcom-next-7.2-20260826.1 -> +qcom-next-20260826.1 / 20260826.1 The first tag of a day deliberately keeps the plain form rather than normalising to ".0". Both comparators read an absent ordinal as lower than a present one, so the respin already sorts above it and every other build is spared a ".0" it never needed. Signed-off-by: Christopher Obbard --- ci/scripts/derive-localversion.sh | 24 ++++++++++++++++-------- debian/rules | 7 ++++--- prepare-source.sh | 18 ++++++++++++------ 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/ci/scripts/derive-localversion.sh b/ci/scripts/derive-localversion.sh index dea72cd7..a5063fe5 100755 --- a/ci/scripts/derive-localversion.sh +++ b/ci/scripts/derive-localversion.sh @@ -12,9 +12,16 @@ set -euo pipefail # carries a variant name and, for branch-tip builds, a hex SHA that can end in # eight digits of its own. # -# For dated tag builds (ref ends in -YYYYMMDD): -# Produces +-. -# Example: qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722 +# For dated tag builds (ref ends in -YYYYMMDD, optionally .): +# Produces +-[.]. +# Example: qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722 +# qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1 +# +# The respin ordinal distinguishes a second tag cut on the same day. It is +# carried verbatim rather than normalised, so the first tag of a day stays +# plain +-: systemd compares the separator before the chunk +# behind it, so an absent ordinal already sorts below a present one and no +# build has to spell a ".0". # # For branch-tip builds (ref does not end in a date): # Uses the kernel variant and a short SHA for uniqueness. @@ -47,8 +54,8 @@ set -euo pipefail # Output: # Two KEY=VALUE lines on stdout, in GITHUB_ENV / 'set -a' form: # -# LOCALVERSION=+qcom-next-20260722 -# SNAPSHOT=20260722 +# LOCALVERSION=+qcom-next-20260722.1 +# SNAPSHOT=20260722.1 # # LOCALVERSION always starts with a plus. SNAPSHOT is empty for branch-tip # builds, which have no date; the Debian version then carries no snapshot @@ -83,9 +90,10 @@ done exit 1 } -# Dated tags use a trailing YYYYMMDD snapshot. The matrix selects the tag set; -# the variant supplies the stable package identity used in LOCALVERSION. -if [[ "$REF" =~ -([0-9]{8})$ ]]; then +# Dated tags use a trailing YYYYMMDD snapshot, optionally followed by a respin +# ordinal. The matrix selects the tag set; the variant supplies the stable +# package identity used in LOCALVERSION. +if [[ "$REF" =~ -([0-9]{8}(\.[0-9]+)?)$ ]]; then SNAPSHOT="${BASH_REMATCH[1]}" LOCALVERSION="+${VARIANT}-${SNAPSHOT}" else diff --git a/debian/rules b/debian/rules index cc20c1b9..ba864470 100755 --- a/debian/rules +++ b/debian/rules @@ -51,7 +51,8 @@ KREL_FILE := debian/kernel.release # Required unless KVER is given explicitly. # KVER Full kernelrelease string (uname -r). Overrides LOCALVERSION. # KVER_EXTRA Optional extra suffix appended after LOCALVERSION, e.g. -ci42. -# SNAPSHOT Dated component of the Debian version, e.g. 20260722. +# SNAPSHOT Dated component of the Debian version: YYYYMMDD with an +# optional . ordinal, e.g. 20260722 or 20260722.1. # Empty for branch-tip builds, which have no snapshot date. # DISTRO Target Debian/Ubuntu suite (default: unstable). # SRCPKG Source package name (default: linux-qcom-next). @@ -139,8 +140,8 @@ prepare: # variant name whose shape is caller-supplied, and for branch-tip builds a # 12-hex SHA that can itself end in eight digits. if [ -n "$(SNAPSHOT)" ]; then \ - echo "$(SNAPSHOT)" | grep -qE '^[0-9]{8}$$' || { \ - echo "ERROR: SNAPSHOT must be YYYYMMDD (got '$(SNAPSHOT)')"; \ + echo "$(SNAPSHOT)" | grep -qE '^[0-9]{8}(\.[0-9]+)?$$' || { \ + echo "ERROR: SNAPSHOT must be YYYYMMDD with an optional . (got '$(SNAPSHOT)')"; \ exit 1; \ }; \ UPSTREAM_VER="$${BASE_KVER_CLEAN}+$(SNAPSHOT)"; \ diff --git a/prepare-source.sh b/prepare-source.sh index 01cfc838..bda24b34 100755 --- a/prepare-source.sh +++ b/prepare-source.sh @@ -54,10 +54,11 @@ OPTIONS: --localversion SUFFIX LOCALVERSION suffix appended to the base kernel version (e.g. +qcom-next-20260722). Auto-detected from git tag if not specified. - --snapshot SNAPSHOT Dated component of the Debian version, e.g. - 20260722. Auto-detected from git tag alongside - --localversion; pass it explicitly whenever - --localversion is passed explicitly. + --snapshot SNAPSHOT Dated component of the Debian version: YYYYMMDD + with an optional . ordinal (e.g. + 20260722 or 20260722.1). Auto-detected from git + tag alongside --localversion; pass it explicitly + whenever --localversion is passed explicitly. --kver-extra SUFFIX Extra suffix appended to the final KVER (e.g. -ci42). @@ -159,14 +160,19 @@ VALID_DISTROS=(noble questing resolute trixie forky sid unstable) [[ -d "$DEBIAN_DIR" ]] || { log_error "Debian dir not found: $DEBIAN_DIR"; exit 1; } # ── Helper: derive LOCALVERSION and SNAPSHOT from a tag name ───────────────── -# qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722 / 20260722 +# qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722 / 20260722 +# qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1 / 20260722.1 +# +# The trailing component is a YYYYMMDD snapshot with an optional respin ordinal +# for a second tag cut on the same day. Matching the date width explicitly (and +# not just "trailing digits") keeps the ordinal attached to it. # # Both fields come out of the tag together. Recovering the snapshot from # LOCALVERSION afterwards would mean parsing a string that also holds a variant # name and, for a branch tip, a hex SHA that can end in eight digits. _auto_version_fields() { local tag="$1" - if [[ "$tag" =~ ^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]+)$ ]]; then + if [[ "$tag" =~ ^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]{8}(\.[0-9]+)?)$ ]]; then LOCALVERSION="+${BASH_REMATCH[1]}-${BASH_REMATCH[2]}" SNAPSHOT="${BASH_REMATCH[2]}" else From 43c37090d30928467c1dac069dc4e359f6da85b8 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 05/71] Spell the Debian snapshot in Debian's own +git idiom The dated component was appended bare, as +20260722, which reads as part of the upstream version rather than as a snapshot marker. Debian spells a VCS snapshot +git, so use that: 7.2.0~rc3+git20260722-0qcom1 7.2.0~rc3+git20260722.1-0qcom1 Ordering is unchanged -- the 'git' is a constant prefix on both sides of every comparison, and the date behind it still decides. Signed-off-by: Christopher Obbard --- debian/rules | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debian/rules b/debian/rules index ba864470..98b12a8e 100755 --- a/debian/rules +++ b/debian/rules @@ -76,7 +76,7 @@ KREL_FILE := debian/kernel.release # debian/localversion Plain file containing the LOCALVERSION suffix (e.g. +qcom-next-20260722). # Read by override_dh_auto_build to pass LOCALVERSION= to make. # debian/pkgversion Plain file containing the full Debian package version -# (e.g. 7.2.0-rc3+20260722-0qcom1). +# (e.g. 7.2.0~rc3+git20260722-0qcom1). # Read by override_dh_gencontrol. # --------------------------------------------------------------------------- KVER_EXTRA ?= @@ -117,9 +117,9 @@ prepare: fi; \ \ # ── Derive package version ─────────────────────────────────────────────── - # Package version format: +- + # Package version format: +git- # where base_kver = kernel version without the LOCALVERSION suffix, and - # snapshot = the SNAPSHOT input. + # snapshot = the SNAPSHOT input (YYYYMMDD with an optional respin ordinal). # For branch-tip builds, which have no snapshot, version = -. # # Extract base_kver: read directly from kernel Makefile (same as above but @@ -144,7 +144,7 @@ prepare: echo "ERROR: SNAPSHOT must be YYYYMMDD with an optional . (got '$(SNAPSHOT)')"; \ exit 1; \ }; \ - UPSTREAM_VER="$${BASE_KVER_CLEAN}+$(SNAPSHOT)"; \ + UPSTREAM_VER="$${BASE_KVER_CLEAN}+git$(SNAPSHOT)"; \ else \ UPSTREAM_VER="$${BASE_KVER_CLEAN}"; \ fi; \ From 396b22994da99d876e573b7614f323e4790300ad Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 06/71] Name the commit in both version strings The snapshot date identifies the tag, not the commit the tag pointed at when the build was cut. A moved tag then produces two different kernels under one version, indistinguishable in the archive and on a running system. Append the commit to both strings: uname -r 7.2.0-rc7+qcom-next-20260826.1-gabcdef123456 Debian version 7.2.0~rc7+git20260826.1~gabcdef123456-0qli1~bpo13+1 The separators differ, and the difference is load-bearing rather than sloppy. systemd weighs the separator before the chunk behind it, so '-g' leaves an absent respin ordinal sorting below a present one. dpkg instead alternates digit and non-digit runs and reads an exhausted run as lower than a letter, so the same '.' or '-' join would decide before reaching the ordinal: 7.2.0+git20260826.g3f2f3ca1a81e > 7.2.0+git20260826.1.gabcdef123456 putting a respin below the build it respins. '~' sorts below everything including the empty string, so the ordinal is always compared first and the SHA only ever breaks a tie between builds sharing a snapshot. The alternative -- spelling ".0" on every non-respin build -- buys the same ordering at the cost of a redundant ordinal in every version forever. What '~' costs is one reading: the version sorts below the same snapshot without a SHA, as though it preceded it. Nothing occupies that slot, because every snapshot build carries a SHA, so the claim is never tested against a real version. The SHA cannot order anything -- two SHAs have no relation -- so a moved tag still needs its respin ordinal or the packaging revision bumped before users can upgrade onto it. What this buys is that the two builds are never confused for each other. --sha becomes required for every build rather than just branch tips, and is validated as hex, since it is now part of every kernel release rather than a fallback identity for refs that had nothing else. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-deb.yml | 17 +++++--- README.md | 23 ++++++++++- ci/scripts/derive-localversion.sh | 57 +++++++++++++++----------- debian/rules | 26 +++++++++++- prepare-source.sh | 28 +++++++++---- 5 files changed, 110 insertions(+), 41 deletions(-) diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml index c947b8cf..53c38c1d 100644 --- a/.github/workflows/build-kernel-deb.yml +++ b/.github/workflows/build-kernel-deb.yml @@ -59,7 +59,7 @@ on: required: false default: '' kver-extra: - description: 'Advanced: extra package-version suffix (e.g. -ci42)' + description: 'Advanced: extra KVER suffix (e.g. -ci42), appended to the kernel release verbatim' required: false default: '' debug-build: @@ -130,7 +130,7 @@ on: type: string default: '' kver-extra: - description: 'Extra suffix appended to the package version' + description: 'Extra KVER suffix, appended to the kernel release verbatim' type: string default: '' debusine-parent-workspace: @@ -283,16 +283,18 @@ jobs: if [[ -n "$OVERRIDE_LV" ]]; then # An override supplies no snapshot, and the ref it was chosen to # replace cannot be trusted to describe it. The Debian version then - # carries no dated component; prepare-source.sh warns about the same. + # carries no +git; prepare-source.sh warns about the same. echo "Using explicit LOCALVERSION override: $OVERRIDE_LV" echo "LOCALVERSION=$OVERRIDE_LV" >> "$GITHUB_ENV" echo "SNAPSHOT=" >> "$GITHUB_ENV" + echo "GITSHA=" >> "$GITHUB_ENV" else - # Emits LOCALVERSION= and SNAPSHOT= lines, already in GITHUB_ENV form. + # Emits LOCALVERSION=, SNAPSHOT= and GITSHA= lines, already in + # GITHUB_ENV form. FIELDS=$(ci/scripts/derive-localversion.sh \ --variant "$KERNEL_VARIANT_INPUT" \ --ref "$KERNEL_REF" \ - --sha "$(echo "$KERNEL_SHA" | cut -c1-12)") + --sha "$KERNEL_SHA") echo "$FIELDS" >> "$GITHUB_ENV" echo "Derived version fields:" echo "$FIELDS" @@ -407,8 +409,13 @@ jobs: --debian-revision "$DEBIAN_REVISION" ) + # All three version fields come from derive-localversion.sh, which + # composed them from the ref and HEAD together. Nothing here appends + # to them: the SHA is already inside LOCALVERSION, and GITSHA carries + # it separately for the Debian version, which joins it differently. [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") [[ -n "$SNAPSHOT" ]] && ARGS+=(--snapshot "$SNAPSHOT") + [[ -n "$GITSHA" ]] && ARGS+=(--git-sha "$GITSHA") [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") # Always passed, unlike the optional inputs above: an empty list is a diff --git a/README.md b/README.md index bcb15a68..bc7fd8d5 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ Supporting scripts keep workflow YAML small and testable: | --- | --- | | `ci/scripts/resolve-matrix.sh` | Validates and flattens matrix rows. | | `ci/scripts/resolve-kernel-ref.sh` | Resolves a matrix-selected dated tag or validates a direct ref. | -| `ci/scripts/derive-localversion.sh` | Derives the version fields from the variant and resolved kernel ref, printing `LOCALVERSION=` and `SNAPSHOT=` lines. `SNAPSHOT` is the dated component of the Debian version and is empty for branch-tip builds. | +| `ci/scripts/derive-localversion.sh` | Derives the version fields from the variant, resolved kernel ref and HEAD, printing `LOCALVERSION=`, `SNAPSHOT=` and `GITSHA=` lines. `SNAPSHOT` is the dated component of the Debian version and is empty for branch-tip builds. | | `ci/scripts/derive-debian-revision.sh` | Derives the final suite-specific `debian_revision` from `debian_version_stub`, `suite_suffix_mapping`, and delivery type. | ## Architecture @@ -399,6 +399,27 @@ For the current matrix, package generation produces: names. Only the Debian version field converts it to `~rcN`, so a release candidate correctly sorts before the corresponding final kernel release. +Every build names both its snapshot and the commit it was cut from: + +| | Format | Example | +| --- | --- | --- | +| Kernel release (`uname -r`) | `+-[.]-g` | `7.2.0-rc7+qcom-next-20260826.1-g011a82096bee` | +| Debian version | `+git[.]~g-` | `7.2.0~rc7+git20260826.1~g011a82096bee-0qli1~bpo13+1` | + +The date orders builds, the respin ordinal separates two tags cut on the same +day, and the SHA is only a final discriminator so a moved tag cannot produce two +different kernels under one version. Branch-tip builds have no date and are +identified by the SHA alone (`7.2.0-rc7+qcom-next-g011a82096bee`). + +The two strings join the SHA differently because their comparators differ. +`uname -r` is compared by systemd, which weighs the separator before the chunk +behind it, so `-g` leaves an absent respin ordinal sorting below a present +one. dpkg instead reads an exhausted run as lower than a letter, which would put +a respin *below* the build it respins, so the Debian version joins with `~` — +lower than everything, including the empty string. The visible cost is that a +Debian version reads as preceding the same snapshot without a SHA; nothing ever +occupies that slot, since every snapshot build carries one. + `KVER_EXTRA` is supported for explicit suffixes such as `-ci42` or `-local`. The packaging rules verify that the declared versioned image package matches the resolved kernel release and fail instead of creating inconsistent metadata. diff --git a/ci/scripts/derive-localversion.sh b/ci/scripts/derive-localversion.sh index a5063fe5..03b45411 100755 --- a/ci/scripts/derive-localversion.sh +++ b/ci/scripts/derive-localversion.sh @@ -5,17 +5,17 @@ set -euo pipefail # Derive the version fields for a build from a kernel variant and resolved ref. # -# Emits LOCALVERSION (the kernel release suffix) and SNAPSHOT (the dated -# component of the Debian version), both derived from the ref in one place. -# SNAPSHOT is emitted alongside rather than recovered from LOCALVERSION later: -# reading it back out means guessing where the date ends in a string that also -# carries a variant name and, for branch-tip builds, a hex SHA that can end in -# eight digits of its own. +# Emits LOCALVERSION (the kernel release suffix), SNAPSHOT (the dated component +# of the Debian version) and GITSHA, all derived from the ref in one place. +# SNAPSHOT and GITSHA are emitted alongside rather than recovered from +# LOCALVERSION later: reading them back out means guessing where each field ends +# in a string that also carries a variant name, and a hex SHA can end in eight +# digits of its own. # # For dated tag builds (ref ends in -YYYYMMDD, optionally .): -# Produces +-[.]. -# Example: qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722 -# qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1 +# Produces +-[.]-g<12 hex>. +# Example: qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722-g07f50dc44edd +# qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1-g07f50dc44edd # # The respin ordinal distinguishes a second tag cut on the same day. It is # carried verbatim rather than normalised, so the first tag of a day stays @@ -23,10 +23,12 @@ set -euo pipefail # behind it, so an absent ordinal already sorts below a present one and no # build has to spell a ".0". # +# The SHA names the commit the tag pointed at when the build was cut, so a +# moved tag cannot silently produce two different kernels under one release. +# # For branch-tip builds (ref does not end in a date): -# Uses the kernel variant and a short SHA for uniqueness. +# Uses the kernel variant and the SHA alone; there is no date to order by. # Example: qcom-next @ 07f50dc44edd -> +qcom-next-g07f50dc44edd -# --sha is required for branch-tip builds. # # Why the leading '+' and not '-': # The suffix ends up in KERNELRELEASE (uname -r), which is the 'version' field @@ -43,19 +45,20 @@ set -euo pipefail # which spells the release candidate ~rcN and orders correctly either way. # # Usage: -# ci/scripts/derive-localversion.sh --variant qcom-next --ref qcom-next-7.2-rc3-20260722 +# ci/scripts/derive-localversion.sh --variant qcom-next --ref qcom-next-7.2-rc3-20260722 --sha 07f50dc44edd # ci/scripts/derive-localversion.sh --variant arduino --ref main --sha 07f50dc44edd # # Options: # --variant VARIANT Kernel variant identifier. Defaults to qcom-next. # --ref REF Kernel ref (tag name or branch name). Required. -# --sha SHA Short commit SHA (required for branch-tip builds). +# --sha SHA Commit SHA, truncated to 12 hex characters. Required. # # Output: -# Two KEY=VALUE lines on stdout, in GITHUB_ENV / 'set -a' form: +# Three KEY=VALUE lines on stdout, in GITHUB_ENV / 'set -a' form: # -# LOCALVERSION=+qcom-next-20260722.1 +# LOCALVERSION=+qcom-next-20260722.1-g07f50dc44edd # SNAPSHOT=20260722.1 +# GITSHA=07f50dc44edd # # LOCALVERSION always starts with a plus. SNAPSHOT is empty for branch-tip # builds, which have no date; the Debian version then carries no snapshot @@ -63,7 +66,7 @@ set -euo pipefail # # Exit codes: # 0 Success. -# 1 Error (invalid args, branch-tip without --sha). +# 1 Error (invalid args, missing or malformed --sha). VARIANT="qcom-next" REF="" @@ -89,24 +92,28 @@ done echo "ERROR: --variant must use lowercase letters, digits, and internal hyphens" >&2 exit 1 } +# Every build identifies its commit, so --sha is required for all of them, not +# just the branch tips that cannot be identified any other way. +[[ "$SHA" =~ ^[0-9a-f]{12,40}$ ]] || { + echo "ERROR: --sha is required and must be at least 12 lowercase hex characters (got '$SHA')" >&2 + exit 1 +} +# 12 chars is upstream's own abbreviation width in -g, and short enough +# to keep the kernel release readable in a boot menu. +GITSHA="${SHA:0:12}" # Dated tags use a trailing YYYYMMDD snapshot, optionally followed by a respin # ordinal. The matrix selects the tag set; the variant supplies the stable # package identity used in LOCALVERSION. if [[ "$REF" =~ -([0-9]{8}(\.[0-9]+)?)$ ]]; then SNAPSHOT="${BASH_REMATCH[1]}" - LOCALVERSION="+${VARIANT}-${SNAPSHOT}" + LOCALVERSION="+${VARIANT}-${SNAPSHOT}-g${GITSHA}" else - # Branch-tip build: need SHA for uniqueness. - [[ -n "$SHA" ]] || { - echo "ERROR: --sha is required for branch-tip builds (ref '$REF' is not a dated tag)" >&2 - exit 1 - } - # Use first 12 chars of SHA for a compact but unambiguous suffix. - SHORT_SHA="${SHA:0:12}" - LOCALVERSION="+${VARIANT}-g${SHORT_SHA}" + # Branch-tip build: no date, so the SHA is the whole identity. + LOCALVERSION="+${VARIANT}-g${GITSHA}" SNAPSHOT="" fi echo "LOCALVERSION=${LOCALVERSION}" echo "SNAPSHOT=${SNAPSHOT}" +echo "GITSHA=${GITSHA}" diff --git a/debian/rules b/debian/rules index 98b12a8e..f5ea1914 100755 --- a/debian/rules +++ b/debian/rules @@ -54,6 +54,9 @@ KREL_FILE := debian/kernel.release # SNAPSHOT Dated component of the Debian version: YYYYMMDD with an # optional . ordinal, e.g. 20260722 or 20260722.1. # Empty for branch-tip builds, which have no snapshot date. +# GITSHA Commit the build was cut from, 12 hex characters. Appended to +# the snapshot as a final discriminator. Ignored without a +# SNAPSHOT to discriminate. # DISTRO Target Debian/Ubuntu suite (default: unstable). # SRCPKG Source package name (default: linux-qcom-next). # BINPKG Binary metapackage name (default: linux-image-qcom-next). @@ -81,6 +84,7 @@ KREL_FILE := debian/kernel.release # --------------------------------------------------------------------------- KVER_EXTRA ?= SNAPSHOT ?= +GITSHA ?= DISTRO ?= unstable SRCPKG ?= linux-qcom-next BINPKG ?= linux-image-qcom-next @@ -117,11 +121,24 @@ prepare: fi; \ \ # ── Derive package version ─────────────────────────────────────────────── - # Package version format: +git- + # Package version format: +git~g- # where base_kver = kernel version without the LOCALVERSION suffix, and # snapshot = the SNAPSHOT input (YYYYMMDD with an optional respin ordinal). # For branch-tip builds, which have no snapshot, version = -. # + # The snapshot orders these; the SHA only discriminates two builds that + # share one. It is joined with '~' rather than '.' because dpkg alternates + # digit and non-digit runs and reads an exhausted run as lower than a + # letter, so a '.' would put a respin BELOW the build it respins: + # + # 7.2.0+git20260826.g3f2f3ca1a81e > 7.2.0+git20260826.1.gabcdef123456 + # + # '~' sorts below everything, including the empty string, so the ordinal is + # always compared before the SHA is reached and no build has to spell a + # ".0" to hold the position. The cost is that a version reads as preceding + # the same snapshot without a SHA -- a slot nothing ever occupies, since + # every snapshot build carries one. + # # Extract base_kver: read directly from kernel Makefile (same as above but # without LOCALVERSION appended), so it is always the pure kernel version. VER=$$(grep '^VERSION = ' Makefile | awk '{print $$3}'); \ @@ -145,6 +162,13 @@ prepare: exit 1; \ }; \ UPSTREAM_VER="$${BASE_KVER_CLEAN}+git$(SNAPSHOT)"; \ + if [ -n "$(GITSHA)" ]; then \ + echo "$(GITSHA)" | grep -qE '^[0-9a-f]{12}$$' || { \ + echo "ERROR: GITSHA must be 12 lowercase hex characters (got '$(GITSHA)')"; \ + exit 1; \ + }; \ + UPSTREAM_VER="$${UPSTREAM_VER}~g$(GITSHA)"; \ + fi; \ else \ UPSTREAM_VER="$${BASE_KVER_CLEAN}"; \ fi; \ diff --git a/prepare-source.sh b/prepare-source.sh index bda24b34..3cecaa99 100755 --- a/prepare-source.sh +++ b/prepare-source.sh @@ -59,6 +59,10 @@ OPTIONS: 20260722 or 20260722.1). Auto-detected from git tag alongside --localversion; pass it explicitly whenever --localversion is passed explicitly. + --git-sha SHA Commit the build was cut from, truncated to 12 + hex characters. Discriminates two builds of one + snapshot (a moved tag). Auto-detected from HEAD + alongside --localversion. --kver-extra SUFFIX Extra suffix appended to the final KVER (e.g. -ci42). @@ -121,6 +125,7 @@ SOURCE_DIR="" DISTRO="$DEFAULT_DISTRO" LOCALVERSION="" SNAPSHOT="" +GITSHA="" KVER_EXTRA="" SRCPKG="$DEFAULT_SRCPKG" BINPKG="$DEFAULT_BINPKG" @@ -134,6 +139,7 @@ while [[ $# -gt 0 ]]; do -d|--distro) DISTRO="$2"; shift 2 ;; --localversion) LOCALVERSION="$2"; shift 2 ;; --snapshot) SNAPSHOT="$2"; shift 2 ;; + --git-sha) GITSHA="$2"; shift 2 ;; --kver-extra) KVER_EXTRA="$2"; shift 2 ;; --srcpkg) SRCPKG="$2"; shift 2 ;; --binpkg) BINPKG="$2"; shift 2 ;; @@ -159,34 +165,36 @@ VALID_DISTROS=(noble questing resolute trixie forky sid unstable) [[ -d "$DEBIAN_DIR" ]] || { log_error "Debian dir not found: $DEBIAN_DIR"; exit 1; } -# ── Helper: derive LOCALVERSION and SNAPSHOT from a tag name ───────────────── -# qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722 / 20260722 -# qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1 / 20260722.1 +# ── Helper: derive LOCALVERSION, SNAPSHOT and GITSHA from a tag name ───────── +# qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722-g / 20260722 +# qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1-g / 20260722.1 # # The trailing component is a YYYYMMDD snapshot with an optional respin ordinal # for a second tag cut on the same day. Matching the date width explicitly (and # not just "trailing digits") keeps the ordinal attached to it. # -# Both fields come out of the tag together. Recovering the snapshot from +# All three fields come out of the tag and HEAD together. Recovering them from # LOCALVERSION afterwards would mean parsing a string that also holds a variant -# name and, for a branch tip, a hex SHA that can end in eight digits. +# name and a hex SHA that can end in eight digits. _auto_version_fields() { local tag="$1" if [[ "$tag" =~ ^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]{8}(\.[0-9]+)?)$ ]]; then - LOCALVERSION="+${BASH_REMATCH[1]}-${BASH_REMATCH[2]}" SNAPSHOT="${BASH_REMATCH[2]}" + LOCALVERSION="+${BASH_REMATCH[1]}-${SNAPSHOT}-g${GITSHA}" else LOCALVERSION="+$tag" SNAPSHOT="" fi } -# ── Auto-detect LOCALVERSION and SNAPSHOT from git tag (if not provided) ───── +# ── Auto-detect LOCALVERSION, SNAPSHOT and GITSHA from git (if not provided) ── if [[ -z "$LOCALVERSION" ]]; then GIT_TAG=$(git -C "$SOURCE_DIR" describe --tags --exact-match 2>/dev/null || true) if [[ -n "$GIT_TAG" ]]; then + [[ -n "$GITSHA" ]] || \ + GITSHA=$(git -C "$SOURCE_DIR" rev-parse --short=12 HEAD 2>/dev/null || true) _auto_version_fields "$GIT_TAG" - log_info "Auto-detected LOCALVERSION='$LOCALVERSION' SNAPSHOT='$SNAPSHOT' from tag '$GIT_TAG'" + log_info "Auto-detected LOCALVERSION='$LOCALVERSION' SNAPSHOT='$SNAPSHOT' GITSHA='$GITSHA' from tag '$GIT_TAG'" else log_warn "LOCALVERSION not set and no exact git tag found." log_warn "Package will be named linux-image- (no branch/date suffix)." @@ -196,7 +204,7 @@ elif [[ -z "$SNAPSHOT" ]]; then # An explicit --localversion is not parsed for a snapshot; say so rather # than silently dropping the dated component from the Debian version. log_warn "--localversion given without --snapshot: the Debian version will" - log_warn "carry no dated component. Pass --snapshot to supply one." + log_warn "carry no +git component. Pass --snapshot to supply one." fi log_step "Configuration:" @@ -207,6 +215,7 @@ log_info " Binary metapkg: $BINPKG" log_info " Debian revision: $DEBIAN_REVISION" [[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" [[ -n "$SNAPSHOT" ]] && log_info " SNAPSHOT: $SNAPSHOT" +[[ -n "$GITSHA" ]] && log_info " GITSHA: $GITSHA" [[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" [[ -n "$KERNEL_CONFIG" ]] && log_info " Kernel config: $KERNEL_CONFIG" [[ -n "$DKMS_MODULES" ]] && log_info " DKMS modules: $DKMS_MODULES" @@ -321,6 +330,7 @@ log_step "Running debian/rules prepare..." PREPARE_ARGS="DISTRO=$DISTRO SRCPKG=$SRCPKG BINPKG=$BINPKG DEBIAN_REVISION=$DEBIAN_REVISION" [[ -n "$LOCALVERSION" ]] && PREPARE_ARGS="$PREPARE_ARGS LOCALVERSION=$LOCALVERSION" [[ -n "$SNAPSHOT" ]] && PREPARE_ARGS="$PREPARE_ARGS SNAPSHOT=$SNAPSHOT" +[[ -n "$GITSHA" ]] && PREPARE_ARGS="$PREPARE_ARGS GITSHA=$GITSHA" [[ -n "$KVER_EXTRA" ]] && PREPARE_ARGS="$PREPARE_ARGS KVER_EXTRA=$KVER_EXTRA" # Spaces are stripped so a list written as "kgsl, camx" stays a single make # argument; debian/rules validates the names it is given. From 08b2a67c1300d2ff494f1505dd3c1b764b9815a7 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 07/71] Date branch-tip builds by their HEAD commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit d6c478d, reworked) A branch-tip build had no snapshot, so debian/rules emitted a version with no +git at all -- 7.2.0~rc7-0qli1~bpo13+1 -- which sorts below every dated build rather than among them. Take the date from the HEAD commit instead, so these builds land in the same sequence: 7.2.0~rc7+git20260821~g07f50dc44edd-… tag build 7.2.0~rc7+git20260904~g07f50dc44edd-… branch tip, dated by commit The date is the committer date of HEAD, not the build date: - rebuilding a commit reproduces its version instead of inventing a higher one on each run - the date describes the source rather than when CI happened to run - it lands in the same space as upstream's tag dates, which track the commit each tag is cut from Committer date rather than author date, because an author date can be months old on a backported patch. Normalised to UTC, since --date=short would otherwise render in each committer's timezone and make one commit yield different dates. The trade-off is that a commit date need not advance: rewinding the branch to an older commit produces a lower version that apt will not offer as an upgrade. Documented in derive-localversion.sh rather than left to be found. The original replaced --sha with --date, because at the time the SHA was not in the version and dating a build meant giving up identifying it. Both are kept here: --sha is required for every build and --date only supplies what a branch tip's ref cannot, so a branch-tip release now has the same shape as a tag build rather than a second shape of its own. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-deb.yml | 26 ++++++++++++-- ci/scripts/derive-localversion.sh | 50 ++++++++++++++++++++------ 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml index 53c38c1d..d31dd3ba 100644 --- a/.github/workflows/build-kernel-deb.yml +++ b/.github/workflows/build-kernel-deb.yml @@ -273,6 +273,18 @@ jobs: echo "KERNEL_SHA=$KERNEL_SHA" >> "$GITHUB_ENV" echo "Kernel HEAD SHA: $KERNEL_SHA" + # Committer date of HEAD, normalised to UTC. Committer rather than + # author date: an author date can be months old on a backported + # patch, which would date the build by when someone first wrote the + # code rather than by when this snapshot came to exist. + KERNEL_DATE=$(TZ=UTC git -C "$KERNEL_DIR" log -1 --format=%cd --date=format-local:%Y%m%d) + [[ "$KERNEL_DATE" =~ ^[0-9]{8}$ ]] || { + echo "::error::could not read a YYYYMMDD commit date from $KERNEL_REF (got '$KERNEL_DATE')" + exit 1 + } + echo "KERNEL_DATE=$KERNEL_DATE" >> "$GITHUB_ENV" + echo "Kernel HEAD commit date (UTC): $KERNEL_DATE" + - name: Derive version fields env: LOCALVERSION_INPUT: ${{ inputs.localversion }} @@ -291,10 +303,17 @@ jobs: else # Emits LOCALVERSION=, SNAPSHOT= and GITSHA= lines, already in # GITHUB_ENV form. + # + # --date is only consulted for branch-tip builds, where the ref + # carries no snapshot of its own. It comes from the commit rather + # than the clock, so rebuilding a commit reproduces its version + # instead of inventing a higher one, and it lands in the same space + # as upstream's tag dates. FIELDS=$(ci/scripts/derive-localversion.sh \ --variant "$KERNEL_VARIANT_INPUT" \ --ref "$KERNEL_REF" \ - --sha "$KERNEL_SHA") + --sha "$KERNEL_SHA" \ + --date "$KERNEL_DATE") echo "$FIELDS" >> "$GITHUB_ENV" echo "Derived version fields:" echo "$FIELDS" @@ -415,7 +434,10 @@ jobs: # it separately for the Debian version, which joins it differently. [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") [[ -n "$SNAPSHOT" ]] && ARGS+=(--snapshot "$SNAPSHOT") - [[ -n "$GITSHA" ]] && ARGS+=(--git-sha "$GITSHA") + # Full SHA: prepare-source.sh takes the first 12 for the version + # strings and records the whole thing in the changelog, alongside the + # repository and ref that the version strings do not name. + ARGS+=(--git-sha "$KERNEL_SHA" --git-clone "$KERNEL_URL" --git-ref "$KERNEL_REF") [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") # Always passed, unlike the optional inputs above: an empty list is a diff --git a/ci/scripts/derive-localversion.sh b/ci/scripts/derive-localversion.sh index 03b45411..eca3c989 100755 --- a/ci/scripts/derive-localversion.sh +++ b/ci/scripts/derive-localversion.sh @@ -27,8 +27,24 @@ set -euo pipefail # moved tag cannot silently produce two different kernels under one release. # # For branch-tip builds (ref does not end in a date): -# Uses the kernel variant and the SHA alone; there is no date to order by. -# Example: qcom-next @ 07f50dc44edd -> +qcom-next-g07f50dc44edd +# Takes the date from the HEAD commit instead of the tag, so the result has +# the same shape as a tag build and orders in the same sequence. +# Example: qcom-next @ 07f50dc44edd, committed 2026-09-04 +# -> +qcom-next-20260904-g07f50dc44edd +# --date is required for these; pass YYYYMMDD.N to separate two branch-tip +# builds sharing a commit date. +# +# The caller supplies the COMMIT date rather than the build date, so that +# rebuilding a commit reproduces its version instead of inventing a higher +# one, and so that the date describes the source rather than when CI ran. It +# also lands in the same space as upstream's tag dates, which track the +# commit each tag is cut from. +# +# The trade-off: a build date always advances, a commit date need not. If the +# branch is ever rewound to an older commit, the next build's version goes +# DOWN and apt will not offer it as an upgrade. That is arguably honest +# -- older source, older version -- but it is the one case where dating by +# the clock would behave differently. # # Why the leading '+' and not '-': # The suffix ends up in KERNELRELEASE (uname -r), which is the 'version' field @@ -46,12 +62,15 @@ set -euo pipefail # # Usage: # ci/scripts/derive-localversion.sh --variant qcom-next --ref qcom-next-7.2-rc3-20260722 --sha 07f50dc44edd -# ci/scripts/derive-localversion.sh --variant arduino --ref main --sha 07f50dc44edd +# ci/scripts/derive-localversion.sh --variant arduino --ref main --sha 07f50dc44edd --date 20260904 # # Options: # --variant VARIANT Kernel variant identifier. Defaults to qcom-next. # --ref REF Kernel ref (tag name or branch name). Required. # --sha SHA Commit SHA, truncated to 12 hex characters. Required. +# --date DATE HEAD commit date as YYYYMMDD or YYYYMMDD.N. Required for +# branch-tip builds; ignored for dated tags, which carry +# their own date. # # Output: # Three KEY=VALUE lines on stdout, in GITHUB_ENV / 'set -a' form: @@ -60,17 +79,17 @@ set -euo pipefail # SNAPSHOT=20260722.1 # GITSHA=07f50dc44edd # -# LOCALVERSION always starts with a plus. SNAPSHOT is empty for branch-tip -# builds, which have no date; the Debian version then carries no snapshot -# component at all. +# LOCALVERSION always starts with a plus. Every build carries a snapshot, +# whether it came from the tag or from the HEAD commit. # # Exit codes: # 0 Success. -# 1 Error (invalid args, missing or malformed --sha). +# 1 Error (invalid args, malformed --sha, branch tip without --date). VARIANT="qcom-next" REF="" SHA="" +DATE="" usage() { sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' @@ -82,6 +101,7 @@ while [[ $# -gt 0 ]]; do --variant) VARIANT="$2"; shift 2 ;; --ref) REF="$2"; shift 2 ;; --sha) SHA="$2"; shift 2 ;; + --date) DATE="$2"; shift 2 ;; -h|--help) usage ;; *) echo "ERROR: Unknown option: $1" >&2; usage ;; esac @@ -109,9 +129,19 @@ if [[ "$REF" =~ -([0-9]{8}(\.[0-9]+)?)$ ]]; then SNAPSHOT="${BASH_REMATCH[1]}" LOCALVERSION="+${VARIANT}-${SNAPSHOT}-g${GITSHA}" else - # Branch-tip build: no date, so the SHA is the whole identity. - LOCALVERSION="+${VARIANT}-g${GITSHA}" - SNAPSHOT="" + # Branch-tip build: the ref carries no date, so the commit date supplies + # one. Without it these builds had no snapshot at all, which put their + # Debian version below every dated build rather than among them. + [[ -n "$DATE" ]] || { + echo "ERROR: --date is required for branch-tip builds (ref '$REF' is not a dated tag)" >&2 + exit 1 + } + [[ "$DATE" =~ ^[0-9]{8}(\.[0-9]+)?$ ]] || { + echo "ERROR: --date must be YYYYMMDD or YYYYMMDD.N (got '$DATE')" >&2 + exit 1 + } + SNAPSHOT="$DATE" + LOCALVERSION="+${VARIANT}-${SNAPSHOT}-g${GITSHA}" fi echo "LOCALVERSION=${LOCALVERSION}" From 2bc171c670dc9acb42387f94db5649efdee720c4 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 08/71] Record the source repository, ref and commit in the changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit d6c478d, reworked) @GITCLONE@ and @BRANCH@ were substituted with empty strings, so every changelog carried a bare "* Source:" naming nothing. Thread the clone URL, ref and full SHA through prepare-source.sh so the entry says where the build came from: * Source: https://github.com/qualcomm-linux/kernel qcom-next * Commit: 07f50dc44edd… The original justified this as the only place the exact source was pinned, since LOCALVERSION identified a build by date alone. That is no longer true -- the version strings carry the commit -- but the repository and ref are still nowhere else, and a 12-character abbreviation is not the commit itself. The comments asserting otherwise are corrected rather than carried over. prepare-source.sh gains one --git-sha taking the full SHA, not two. The original added a second option under that name for the changelog, alongside the existing one for the version field; bash takes the first matching case arm, so the changelog copy would have been silently unset and every entry would have read "Commit: unknown". They are the same commit at two widths, so the short form is derived from the long one in a single place. Signed-off-by: Christopher Obbard --- debian/changelog.in | 1 + debian/rules | 16 ++++++++++++++-- prepare-source.sh | 35 +++++++++++++++++++++++++++-------- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/debian/changelog.in b/debian/changelog.in index 95b41aeb..802a600a 100644 --- a/debian/changelog.in +++ b/debian/changelog.in @@ -3,6 +3,7 @@ * Qualcomm ARM64 Linux kernel package * Kernel version: @KVER@ * Source: @GITCLONE@ @BRANCH@ + * Commit: @GITSHA@ * Includes kernel image, modules, DTBs, headers, and debug symbols -- Bjordis Collaku Fri, 14 Nov 2025 10:00:00 -0800 diff --git a/debian/rules b/debian/rules index f5ea1914..d888c1c0 100755 --- a/debian/rules +++ b/debian/rules @@ -67,6 +67,11 @@ KREL_FILE := debian/kernel.release # nothing. Each name must have a -dkms package available # in the build environment; the Build-Depends entry for it is # generated from this list, so nothing else declares it. +# GIT_CLONE Kernel repository URL, recorded in debian/changelog. +# GIT_REF Resolved kernel ref (tag or branch), recorded in the changelog. +# GIT_SHA Full kernel HEAD commit SHA, recorded in the changelog. +# GITSHA above is the same commit at 12 characters, for the +# version strings; this is the unabbreviated value. # # Outputs (generated files, all listed in debian/clean): # debian/control Substituted from debian/control.in @@ -90,6 +95,12 @@ SRCPKG ?= linux-qcom-next BINPKG ?= linux-image-qcom-next DEBIAN_REVISION ?= 0qcom1 DKMS_MODULES ?= +# Source provenance recorded in debian/changelog. The version strings name the +# commit but not the repository it came from, so without these a build cannot be +# traced back to its source tree. +GIT_CLONE ?= unknown +GIT_REF ?= unknown +GIT_SHA ?= unknown .PHONY: prepare prepare: @@ -227,8 +238,9 @@ prepare: -e "s|@PKGVER@|$$PKG_VERSION|g" \ -e "s|@DISTRO@|$(DISTRO)|g" \ -e "s|@KVER@|$$KVER_RESOLVED|g" \ - -e "s|@GITCLONE@||g" \ - -e "s|@BRANCH@||g" \ + -e "s|@GITCLONE@|$(GIT_CLONE)|g" \ + -e "s|@BRANCH@|$(GIT_REF)|g" \ + -e "s|@GITSHA@|$(GIT_SHA)|g" \ debian/changelog.in > debian/changelog; \ \ # ── Write runtime state files ───────────────────────────────────────────── diff --git a/prepare-source.sh b/prepare-source.sh index 3cecaa99..b6219431 100755 --- a/prepare-source.sh +++ b/prepare-source.sh @@ -59,12 +59,16 @@ OPTIONS: 20260722 or 20260722.1). Auto-detected from git tag alongside --localversion; pass it explicitly whenever --localversion is passed explicitly. - --git-sha SHA Commit the build was cut from, truncated to 12 - hex characters. Discriminates two builds of one - snapshot (a moved tag). Auto-detected from HEAD - alongside --localversion. + --git-sha SHA Full commit SHA the build was cut from. Its + first 12 characters discriminate two builds of + one snapshot (a moved tag) in the version + strings; the full value is recorded in the + changelog. Auto-detected from HEAD. --kver-extra SUFFIX Extra suffix appended to the final KVER (e.g. -ci42). + --git-clone URL Kernel repository URL, recorded in the changelog. + --git-ref REF Resolved kernel ref (tag or branch), recorded in + the changelog. Package naming: --srcpkg NAME Source package name (default: $DEFAULT_SRCPKG) @@ -125,13 +129,15 @@ SOURCE_DIR="" DISTRO="$DEFAULT_DISTRO" LOCALVERSION="" SNAPSHOT="" -GITSHA="" KVER_EXTRA="" SRCPKG="$DEFAULT_SRCPKG" BINPKG="$DEFAULT_BINPKG" DEBIAN_REVISION="$DEFAULT_DEBIAN_REVISION" KERNEL_CONFIG="" DKMS_MODULES="" +GIT_CLONE="" +GIT_REF="" +GIT_SHA="" while [[ $# -gt 0 ]]; do case $1 in @@ -139,8 +145,10 @@ while [[ $# -gt 0 ]]; do -d|--distro) DISTRO="$2"; shift 2 ;; --localversion) LOCALVERSION="$2"; shift 2 ;; --snapshot) SNAPSHOT="$2"; shift 2 ;; - --git-sha) GITSHA="$2"; shift 2 ;; + --git-sha) GIT_SHA="$2"; shift 2 ;; --kver-extra) KVER_EXTRA="$2"; shift 2 ;; + --git-clone) GIT_CLONE="$2"; shift 2 ;; + --git-ref) GIT_REF="$2"; shift 2 ;; --srcpkg) SRCPKG="$2"; shift 2 ;; --binpkg) BINPKG="$2"; shift 2 ;; --debian-revision) DEBIAN_REVISION="$2"; shift 2 ;; @@ -165,6 +173,13 @@ VALID_DISTROS=(noble questing resolute trixie forky sid unstable) [[ -d "$DEBIAN_DIR" ]] || { log_error "Debian dir not found: $DEBIAN_DIR"; exit 1; } +# ── Resolve the commit, once ───────────────────────────────────────────────── +# One SHA, used at two widths: the first 12 characters go in the version strings +# (short enough to keep a boot menu readable), the full value goes in the +# changelog. Deriving one from the other is what keeps them the same commit. +[[ -n "$GIT_SHA" ]] || GIT_SHA=$(git -C "$SOURCE_DIR" rev-parse HEAD 2>/dev/null || true) +GITSHA="${GIT_SHA:0:12}" + # ── Helper: derive LOCALVERSION, SNAPSHOT and GITSHA from a tag name ───────── # qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722-g / 20260722 # qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1-g / 20260722.1 @@ -191,8 +206,6 @@ _auto_version_fields() { if [[ -z "$LOCALVERSION" ]]; then GIT_TAG=$(git -C "$SOURCE_DIR" describe --tags --exact-match 2>/dev/null || true) if [[ -n "$GIT_TAG" ]]; then - [[ -n "$GITSHA" ]] || \ - GITSHA=$(git -C "$SOURCE_DIR" rev-parse --short=12 HEAD 2>/dev/null || true) _auto_version_fields "$GIT_TAG" log_info "Auto-detected LOCALVERSION='$LOCALVERSION' SNAPSHOT='$SNAPSHOT' GITSHA='$GITSHA' from tag '$GIT_TAG'" else @@ -332,6 +345,12 @@ PREPARE_ARGS="DISTRO=$DISTRO SRCPKG=$SRCPKG BINPKG=$BINPKG DEBIAN_REVISION=$DEBI [[ -n "$SNAPSHOT" ]] && PREPARE_ARGS="$PREPARE_ARGS SNAPSHOT=$SNAPSHOT" [[ -n "$GITSHA" ]] && PREPARE_ARGS="$PREPARE_ARGS GITSHA=$GITSHA" [[ -n "$KVER_EXTRA" ]] && PREPARE_ARGS="$PREPARE_ARGS KVER_EXTRA=$KVER_EXTRA" +# Source provenance for debian/changelog. LOCALVERSION identifies a build by +# date, not by commit, so the SHA recorded here is what makes a build traceable +# back to exact source -- particularly for branch-tip builds. +[[ -n "$GIT_CLONE" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_CLONE=$GIT_CLONE" +[[ -n "$GIT_REF" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_REF=$GIT_REF" +[[ -n "$GIT_SHA" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_SHA=$GIT_SHA" # Spaces are stripped so a list written as "kgsl, camx" stays a single make # argument; debian/rules validates the names it is given. [[ -n "$DKMS_MODULES" ]] && PREPARE_ARGS="$PREPARE_ARGS DKMS_MODULES=$(tr -d ' ' <<< "$DKMS_MODULES")" From a31a5d7fc0cc7d51bab80b56319c3fb36d260e8e Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 09/71] docs: describe the version scheme Records why the kernel release and the Debian version are two strings rather than one, and why each spells the same four fields differently: the '+' join that keeps a release candidate below its final release under systemd, and the '~' before the SHA that keeps a respin above the build it respins under dpkg. Both separators look arbitrary and both are load-bearing, so the reasoning is written down where someone tempted to normalise them will find it. Also covers what the version deliberately does not carry -- the repository and ref live in the changelog -- and the one case where a branch-tip version can go backwards, when the branch is rewound to an older commit. README.md loses its copy of the same reasoning, which had grown into a second account that would drift, and keeps the format table plus a pointer here. Scoped to qcom-next; the derivation is not variant-agnostic. Signed-off-by: Christopher Obbard --- README.md | 23 ++----- docs/version.md | 179 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 16 deletions(-) create mode 100644 docs/version.md diff --git a/README.md b/README.md index bc7fd8d5..294e6f48 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,9 @@ isolated `kernel_variant + suite` build leg. Both build the same kernel ref. `derive-localversion.sh` folds the variant name into LOCALVERSION, so each produces a distinct kernel release -(`-qcom-next-` and `-qcom-next-debug-`) and therefore a distinct -versioned image package that can be installed alongside the other. +(`+qcom-next--g` and `+qcom-next-debug--g`) and therefore a +distinct versioned image package that can be installed alongside the other. See +[docs/version.md](docs/version.md) for how the version strings are composed. `ci/build-matrix.json` is the source of truth; this table is a summary. @@ -238,7 +239,7 @@ Supporting scripts keep workflow YAML small and testable: | --- | --- | | `ci/scripts/resolve-matrix.sh` | Validates and flattens matrix rows. | | `ci/scripts/resolve-kernel-ref.sh` | Resolves a matrix-selected dated tag or validates a direct ref. | -| `ci/scripts/derive-localversion.sh` | Derives the version fields from the variant, resolved kernel ref and HEAD, printing `LOCALVERSION=`, `SNAPSHOT=` and `GITSHA=` lines. `SNAPSHOT` is the dated component of the Debian version and is empty for branch-tip builds. | +| `ci/scripts/derive-localversion.sh` | Derives the version fields from the variant, resolved kernel ref and HEAD, printing `LOCALVERSION=`, `SNAPSHOT=` and `GITSHA=` lines. `SNAPSHOT` is the dated component of the Debian version: the tag's date, or the HEAD commit date for a branch-tip build. Scheme and rationale: [docs/version.md](docs/version.md). | | `ci/scripts/derive-debian-revision.sh` | Derives the final suite-specific `debian_revision` from `debian_version_stub`, `suite_suffix_mapping`, and delivery type. | ## Architecture @@ -406,19 +407,9 @@ Every build names both its snapshot and the commit it was cut from: | Kernel release (`uname -r`) | `+-[.]-g` | `7.2.0-rc7+qcom-next-20260826.1-g011a82096bee` | | Debian version | `+git[.]~g-` | `7.2.0~rc7+git20260826.1~g011a82096bee-0qli1~bpo13+1` | -The date orders builds, the respin ordinal separates two tags cut on the same -day, and the SHA is only a final discriminator so a moved tag cannot produce two -different kernels under one version. Branch-tip builds have no date and are -identified by the SHA alone (`7.2.0-rc7+qcom-next-g011a82096bee`). - -The two strings join the SHA differently because their comparators differ. -`uname -r` is compared by systemd, which weighs the separator before the chunk -behind it, so `-g` leaves an absent respin ordinal sorting below a present -one. dpkg instead reads an exhausted run as lower than a letter, which would put -a respin *below* the build it respins, so the Debian version joins with `~` — -lower than everything, including the empty string. The visible cost is that a -Debian version reads as preceding the same snapshot without a SHA; nothing ever -occupies that slot, since every snapshot build carries one. +The two strings spell the same fields differently because they are compared by +different rules — `+` and `~` are both load-bearing, not stylistic. See +[docs/version.md](docs/version.md) before changing either. `KVER_EXTRA` is supported for explicit suffixes such as `-ci42` or `-local`. The packaging rules verify that the declared versioned image package matches the diff --git a/docs/version.md b/docs/version.md new file mode 100644 index 00000000..f396079d --- /dev/null +++ b/docs/version.md @@ -0,0 +1,179 @@ +# Version scheme + +This describes how a build's two version strings are composed. The examples are +`qcom-next` specific: another variant substitutes its own name and Debian +revision stub, and a variant built from a differently-shaped tag would need its +own derivation. + +A build produces two version strings, and they are deliberately not the same +string: + +```text +uname -r 7.2.0-rc7+qcom-next-20260821-gabcdef123456 +Debian version 7.2.0~rc7+git20260821~gabcdef123456-0qli1~bpo13+1 +``` + +They carry the same four facts — upstream kernel version, snapshot date, +same-day respin, commit — but they are read by two different comparators with +two different sets of rules, and each string is spelled for its own. + +## What the fields mean + +Both strings are derived from the tag and HEAD together, by +`ci/scripts/derive-localversion.sh`: + +```text +qcom-next-7.2-rc7-20260821 @ abcdef123456… + │ │ + │ └── snapshot: 20260821, optionally . + └────────── upstream kernel version: 7.2.0-rc7 +``` + +The **snapshot date** orders builds. The **respin ordinal** separates two tags +cut on the same day. The **commit** orders nothing — two SHAs have no relation — +and exists only so that a moved tag cannot produce two different kernels under +one version. It therefore comes last in both strings, after everything that does +carry ordering. + +There is no `.0` on the first tag of a day. Both comparators read an absent +ordinal as lower than a present one, so the respin already sorts above it, and +the version stays closer to the tag it came from. + +## Kernel release + +```text ++qcom-next-[.]-g<12 hex> +``` + +The full upstream version survives here, `-rc7` included: `uname -r` is the +first thing in a bug report, and it is what says whether the reporter is on a +release candidate or a stable sublevel. + +The suffix joins with `+`, not `-`. systemd compares the separator before the +chunk behind it, and `-` sorts below `+`, so `+` puts every release candidate +below the final release that follows it. Joining with `-` instead falls through +to a plain comparison of `rc` against `qcom`, where `r` > `q`, and every rc +outranks its own final release in the boot menu. This is the same trick Debian's +own kernels use (`linux-image-7.1.10+deb14-amd64`). + +The variant name is part of the string, so a flavour is a distinct kernel that +installs alongside the others rather than replacing them: + +```text +7.2.0-rc7+qcom-next-debug-20260821-gabcdef123456 +``` + +This string is also the versioned binary package name +(`linux-image-`), so a new commit means a new package name. That +is intended: it is what lets several builds coexist, and what makes the commit +recoverable from an archive listing. + +## Debian version + +```text ++git[.]~g<12 hex>- +``` + +`-rcN` becomes `~rcN`, because dpkg reads `~` as "sorts below", giving +`7.2.0~rc7 < 7.2.0`. Spelled `-rc7` it would sort *above* the release it +precedes. + +The snapshot is spelled `+git` in Debian's usual idiom for a VCS snapshot. + +The commit joins with `~`, and this is the part most likely to look like a typo. +dpkg alternates digit and non-digit runs and reads an exhausted run as lower +than a letter, so joining with `.` decides the comparison before ever reaching +the respin ordinal: + +```text +7.2.0+git20260826.g3f2f3ca1a81e > 7.2.0+git20260826.1.gabcdef123456 +``` + +— the respin sorting *below* the build it respins. `~` sorts below everything, +including the empty string, so the ordinal is always compared first and the SHA +only ever breaks a tie between builds that share a snapshot. The alternative, +spelling `.0` on every non-respin build, buys the same ordering at the cost of a +redundant ordinal in every version forever. + +The cost of `~` is one misleading reading: the version sorts below the same +snapshot without a SHA, as though it preceded it. Nothing occupies that slot, +because every snapshot build carries a SHA. + +The revision (`0qli1~bpo13+1`) is derived separately, from the version stub, the +suite, and whether the build is a Daily or a Release. See the matrix +documentation in the top-level [README](../README.md#matrix-model); the trailing +digit on the stub is the packaging revision, bumped when the packaging changes +but the kernel snapshot does not. + +## Ordering + +The full chain for one suite, in the order dpkg sorts it: + +```text +7.2.0~rc7+git20260820.1~g011a82096bee-0qli1~bpo13+1 first tag of the 20th +7.2.0~rc7+git20260820.2~g3f2f3ca1a81e-0qli1~bpo13+1 respin, same day +7.2.0~rc7+git20260821~gabcdef123456-0qli1~bpo13+1~ Daily, next snapshot +7.2.0~rc7+git20260821~gabcdef123456-0qli1~bpo13+1 Release of the same +7.2.0~rc7+git20260821~gabcdef123456-0qli2~bpo13+1 packaging rebuild +7.2.0+git20260902~g123456789abc-0qli1~bpo13+1 7.2 final +``` + +A Daily sorts below the Release of the same snapshot because its revision ends +in a trailing `~`. + +## Moved tags + +If an upstream tag is re-cut against a different commit, the SHA changes and so +does the version, so the two builds cannot be mistaken for each other. But +nothing guarantees the replacement sorts *above* the original — two hashes have +no order — so apt may not offer it as an upgrade: + +```text +7.2.0~rc7+git20260821~g011a82096bee-0qli1~bpo13+1 original +7.2.0~rc7+git20260821~gabcdef123456-0qli1~bpo13+1 retagged; happens to sort above +7.2.0~rc7+git20260821~g0009f3c1d2e4-0qli1~bpo13+1 retagged; sorts BELOW, no upgrade +``` + +Whether it lands above or below is down to the hex, so treat a moved tag as +needing a version bump of its own — the respin ordinal, or the packaging +revision: + +```text +7.2.0~rc7+git20260821.1~g0009f3c1d2e4-0qli1~bpo13+1 supersedes it either way +``` + +## Branch-tip builds + +A build from a branch rather than a tag has no tag date, so the date comes from +the HEAD commit instead. The result has the same shape as a tag build and orders +in the same sequence: + +```text +uname -r 7.2.0-rc7+qcom-next-20260904-g07f50dc44edd +Debian version 7.2.0~rc7+git20260904~g07f50dc44edd-0qli1~bpo13+1 +``` + +It is the *committer* date of the commit, not the time the build ran. That means +rebuilding a commit reproduces its version instead of inventing a higher one, +and the date describes the source rather than when CI happened to start. Author +dates are not used, because a backported patch can carry one months old. + +The catch is that a build clock only ever moves forwards, and a commit date does +not. If the branch is rewound to an older commit, the next build's version goes +*down*, and apt will not offer it as an upgrade. That is arguably the honest +answer — older source, older version — but it is the one case where dating by +the clock would behave differently. + +## What is not in the version + +The version strings name the commit, but not the tree it came from. The +repository, the resolved ref and the full 40-character SHA are recorded in the +package changelog instead: + +```text + * Kernel version: 7.2.0-rc7+qcom-next-20260904-g07f50dc44edd + * Source: https://github.com/qualcomm-linux/kernel qcom-next + * Commit: 07f50dc44edd… +``` + +so `apt changelog` on an installed image is enough to find the exact source. From 01cd6c2e4b9316bd7925732388456e2f9aa6bb7e Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 10/71] ci: resolve staging build-deps from qli-staging The build job passed a fixed 'EXTRA_BUILD_DEP_WORKSPACES: qli', so every run resolved its build-dependencies from the released qli workspace alone. Daily and PR builds are staged work: they can depend on a package version that has only reached qli-staging, and with qli the sole extra repository those builds fail to satisfy the dependency, or silently pick up the older released one. Pick the workspace list from the same input that already picks the environment and the publish path. A run that promotes to a release workspace is a Production deployment and keeps reading qli only, so a released kernel is never built against something that has not been released alongside it. A run that only builds into a CI workspace is a Staging one and reads qli and qli-staging, matching what the staged archive actually offers. Closes: #93 Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debusine.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-kernel-debusine.yml b/.github/workflows/build-kernel-debusine.yml index 7b8cc17a..b632bf37 100644 --- a/.github/workflows/build-kernel-debusine.yml +++ b/.github/workflows/build-kernel-debusine.yml @@ -12,6 +12,10 @@ name: build-debusine # target-workspace != '' -> release path: promote from CI workspace to the # target Debusine workspace via debusine-action lib/release. # +# The same input picks the GitHub environment and the workspaces the build +# resolves its build-dependencies from: a Staging run also reads qli-staging, +# a Production run reads qli alone. +# # Jobs: # build (debusine-pkg-builder container): generates .dsc, submits to Debusine. # publish (self-hosted runner): S3 upload (daily path only). @@ -120,7 +124,9 @@ jobs: DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} DEBUSINE_PARENT_WORKSPACE: ${{ inputs.debusine-parent-workspace }} SUITE: ${{ inputs.suite }} - EXTRA_BUILD_DEP_WORKSPACES: qli + # Staging runs may build against packages that have not been released + # to qli yet, so they resolve build-dependencies from qli-staging too. + EXTRA_BUILD_DEP_WORKSPACES: ${{ inputs.target-workspace != '' && 'qli' || 'qli qli-staging' }} run: | # No -x here: DEBUSINE_TOKEN is in env, keep xtrace off. set -euo pipefail From 9051a3a424b927dfd9fc54822f7181dc676302cd Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 11/71] ci: name the Dependabot config dependabot.yml The file was committed as .github/dependabots.yaml. Dependabot reads .github/dependabot.yml and nothing else, so the plural name meant the configuration was never read: no version updates have ever been opened for the github-actions ecosystem this repository asks for. Rename it to the name Dependabot expects. The contents are unchanged. Signed-off-by: Christopher Obbard --- .github/{dependabots.yaml => dependabot.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{dependabots.yaml => dependabot.yml} (100%) diff --git a/.github/dependabots.yaml b/.github/dependabot.yml similarity index 100% rename from .github/dependabots.yaml rename to .github/dependabot.yml From a484a5bda801549c90a2008b21aebef22ea44db4 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 12/71] ci: make one matrix entry one generated package The delivery matrix described build legs indirectly. A row carried a list of suites that resolve-matrix.sh flattened into one leg per suite, and each leg's Debian revision was assembled at resolve time from a per-row debian_version_stub, a matrix-wide suite_suffix_mapping, and a suffix implied by the row's type. Reading the matrix told you which packages existed only after you had run the flattening and the revision formula in your head, and the row's debian_version_suffix field existed purely to write half of that formula down again so a validator could check the two agreed. Split the rows so that one entry is one generated package: a single kernel_variant, type and suite, with its debian_revision stated outright. suite_suffix_mapping and the stub/suffix pair are gone, and with them derive-debian-revision.sh, whose whole job was to apply the formula they encoded. There is nothing left to expand or derive at resolve time, so what an entry says is what gets built. The cost is duplication: nine entries repeat their variant's git_clone, srcpkg, binpkg and kernel_config. They are written out in full rather than sharing YAML anchors, so each entry can be read, grepped and changed on its own, and the resolver takes on the invariants that duplication puts at risk. A variant's entries must agree on srcpkg, binpkg and kernel_config, which decide what the package is; its entries of one type must agree on the kernel ref, so a release ref bump cannot skip a suite and quietly ship one suite a different kernel from its siblings; no two entries may build the same srcpkg at the same revision; and a suite's Daily revision must be its Release revision plus the trailing ~ that sorts it below. Move the matrix to YAML and rewrite the resolver in Python. The resolver was 372 lines whose validation was a single jq program built from elif chains, where each check had to fall through to the next and a new rule meant extending an expression rather than adding a function. The Python collects every problem in one pass and reports them together, so a bad matrix yields its full list of errors instead of the first one jq happened to reach. Both files drop out of the change naturally: the schema is being rewritten anyway, and YAML lets each entry carry the comment that explains it. Unknown fields are now rejected at both entry and root level. Anything left over from this schema change - a stray suites: or suite_suffix_mapping: - fails loudly rather than sitting in the file looking authoritative while nothing reads it. The dkms list moves across with the rest of the schema, validated by resolve-matrix.py the same way kernel_config is and joined into the same comma-separated workflow input. Every entry keeps the kgsl, camx and iris-vpu the row it came from carried, including the ones the Ubuntu override in build-kernel-deb.yml still trims at build time; moving that policy into the matrix is a separate change. Verified against the old resolver across all fifteen filter combinations the workflows can produce: the emitted JSON is identical, so every build leg keeps its inputs and its debian_revision. build-kernel-deb.yml's direct-dispatch path loses the mapping it used to consult, so its debian-version-stub input becomes debian-revision. Left empty it looks up the Daily entry for the variant and suite it was given and builds at the revision the daily build would have used. Dispatching a combination the matrix does not configure now fails instead of inventing a version for it; the error names the override that builds it ad hoc. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-deb.yml | 102 ++--- .github/workflows/daily.yml | 45 ++- .github/workflows/pr-build.yml | 10 +- .github/workflows/release.yml | 27 +- .gitignore | 3 + ci/build-matrix.json | 108 ----- ci/build-matrix.yaml | 192 +++++++++ ci/scripts/derive-debian-revision.sh | 131 ------- ci/scripts/resolve-matrix.py | 523 +++++++++++++++++++++++++ ci/scripts/resolve-matrix.sh | 400 ------------------- 10 files changed, 798 insertions(+), 743 deletions(-) delete mode 100644 ci/build-matrix.json create mode 100644 ci/build-matrix.yaml delete mode 100755 ci/scripts/derive-debian-revision.sh create mode 100755 ci/scripts/resolve-matrix.py delete mode 100755 ci/scripts/resolve-matrix.sh diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml index d31dd3ba..58e05bef 100644 --- a/.github/workflows/build-kernel-deb.yml +++ b/.github/workflows/build-kernel-deb.yml @@ -50,16 +50,16 @@ on: description: 'Advanced: out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' required: false default: '' - debian-version-stub: - description: 'Advanced: Debian version stub; the selected suite''s mapped suffix and a Daily-style trailing ~ are applied automatically' + debian-revision: + description: 'Advanced: Debian revision override (empty takes the Daily revision the matrix gives this variant and suite)' required: false - default: '0qli1' + default: '' localversion: description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' required: false default: '' kver-extra: - description: 'Advanced: extra KVER suffix (e.g. -ci42), appended to the kernel release verbatim' + description: 'Advanced: extra package-version suffix (e.g. -ci42)' required: false default: '' debug-build: @@ -124,13 +124,13 @@ on: debian-revision: description: 'Debian revision component of the package version' type: string - default: '0qli1~' + default: '0qli~' localversion: description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' type: string default: '' kver-extra: - description: 'Extra KVER suffix, appended to the kernel release verbatim' + description: 'Extra suffix appended to the package version' type: string default: '' debusine-parent-workspace: @@ -214,7 +214,11 @@ jobs: - name: Checkout pkg-linux-qcom # debian/ and ci/ live on the same branch, so one checkout supplies # both the packaging and resolve-kernel-ref.sh, derive-localversion.sh, - # derive-debian-revision.sh and the suite_suffix_mapping. + # resolve-matrix.py and the delivery matrix it reads. + # + # An empty ref leaves actions/checkout on github.sha, the commit this + # run was dispatched from or the one the calling workflow runs at. + # Credentials are kept: the self-pr step below fetches from origin. uses: actions/checkout@v4 - name: Checkout docker-pkg-build @@ -273,19 +277,7 @@ jobs: echo "KERNEL_SHA=$KERNEL_SHA" >> "$GITHUB_ENV" echo "Kernel HEAD SHA: $KERNEL_SHA" - # Committer date of HEAD, normalised to UTC. Committer rather than - # author date: an author date can be months old on a backported - # patch, which would date the build by when someone first wrote the - # code rather than by when this snapshot came to exist. - KERNEL_DATE=$(TZ=UTC git -C "$KERNEL_DIR" log -1 --format=%cd --date=format-local:%Y%m%d) - [[ "$KERNEL_DATE" =~ ^[0-9]{8}$ ]] || { - echo "::error::could not read a YYYYMMDD commit date from $KERNEL_REF (got '$KERNEL_DATE')" - exit 1 - } - echo "KERNEL_DATE=$KERNEL_DATE" >> "$GITHUB_ENV" - echo "Kernel HEAD commit date (UTC): $KERNEL_DATE" - - - name: Derive version fields + - name: Derive LOCALVERSION env: LOCALVERSION_INPUT: ${{ inputs.localversion }} KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} @@ -293,55 +285,44 @@ jobs: set -euo pipefail OVERRIDE_LV="$LOCALVERSION_INPUT" if [[ -n "$OVERRIDE_LV" ]]; then - # An override supplies no snapshot, and the ref it was chosen to - # replace cannot be trusted to describe it. The Debian version then - # carries no +git; prepare-source.sh warns about the same. echo "Using explicit LOCALVERSION override: $OVERRIDE_LV" echo "LOCALVERSION=$OVERRIDE_LV" >> "$GITHUB_ENV" - echo "SNAPSHOT=" >> "$GITHUB_ENV" - echo "GITSHA=" >> "$GITHUB_ENV" else - # Emits LOCALVERSION=, SNAPSHOT= and GITSHA= lines, already in - # GITHUB_ENV form. - # - # --date is only consulted for branch-tip builds, where the ref - # carries no snapshot of its own. It comes from the commit rather - # than the clock, so rebuilding a commit reproduces its version - # instead of inventing a higher one, and it lands in the same space - # as upstream's tag dates. - FIELDS=$(ci/scripts/derive-localversion.sh \ + LOCALVERSION=$(ci/scripts/derive-localversion.sh \ --variant "$KERNEL_VARIANT_INPUT" \ --ref "$KERNEL_REF" \ - --sha "$KERNEL_SHA" \ - --date "$KERNEL_DATE") - echo "$FIELDS" >> "$GITHUB_ENV" - echo "Derived version fields:" - echo "$FIELDS" + --sha "$(echo "$KERNEL_SHA" | cut -c1-12)") + echo "LOCALVERSION=$LOCALVERSION" >> "$GITHUB_ENV" + echo "Derived LOCALVERSION: $LOCALVERSION" fi - name: Resolve Debian revision - # workflow_call callers (daily.yml, release.yml) pass an already - # fully-derived debian-revision from resolve-matrix.sh. Direct - # workflow_dispatch runs pass only debian-version-stub and get the - # selected suite's mapped suffix applied automatically, always using - # Daily semantics since direct dispatch is build-only and - # non-promoting. + # workflow_call callers (daily.yml, release.yml) pass the entry's own + # debian-revision straight from the matrix. A direct workflow_dispatch + # that leaves it empty looks the same entry up itself, so a manual + # build of a configured variant and suite gets exactly the revision the + # daily build would have used. The lookup is always the Daily entry: + # direct dispatch is build-only and never promotes. env: DEBIAN_REVISION_INPUT: ${{ inputs.debian-revision }} - DEBIAN_VERSION_STUB_INPUT: ${{ inputs.debian-version-stub || '0qli1' }} + KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} run: | set -euo pipefail INPUT_REVISION="$DEBIAN_REVISION_INPUT" if [[ -n "$INPUT_REVISION" ]]; then DEBIAN_REVISION="$INPUT_REVISION" - echo "Using matrix-derived Debian revision: $DEBIAN_REVISION" + echo "Using supplied Debian revision: $DEBIAN_REVISION" else - DEBIAN_REVISION=$(ci/scripts/derive-debian-revision.sh \ - --stub "$DEBIAN_VERSION_STUB_INPUT" \ + DEBIAN_REVISION=$(ci/scripts/resolve-matrix.py \ + --type Daily \ + --kernel-variant "$KERNEL_VARIANT_INPUT" \ --suite "$DISTRO" \ - --delivery-type Daily \ - --matrix-file ci/build-matrix.json) - echo "Derived Debian revision for direct dispatch: $DEBIAN_REVISION" + --field debian_revision) || { + echo "::error::No Daily matrix entry for kernel-variant=$KERNEL_VARIANT_INPUT suite=$DISTRO." + echo "::error::Add one to ci/build-matrix.yaml, or set the debian-revision input to build this combination ad hoc." + exit 1 + } + echo "Matrix Debian revision for $KERNEL_VARIANT_INPUT/$DISTRO: $DEBIAN_REVISION" fi echo "DEBIAN_REVISION=$DEBIAN_REVISION" >> "$GITHUB_ENV" @@ -411,10 +392,10 @@ jobs: # A listed module is a presence contract, so an Ubuntu leg that asked # for them would hard-fail rather than quietly ship without them. # - # This is delivery policy and belongs in ci/build-matrix.json, which - # cannot express a per-suite dkms list yet. Revert this override — - # and the "needs: resolve"/SUITE_FAMILY plumbing that feeds it — once - # the matrix can. + # This is delivery policy and belongs in ci/build-matrix.yaml, whose + # entries are now one per suite and can carry it. Revert this + # override — and the "needs: resolve"/SUITE_FAMILY plumbing that + # feeds it — in the commit that moves the policy there. if [[ "$SUITE_FAMILY" == "ubuntu" ]]; then DKMS_LIST="kgsl" echo "Ubuntu build: DKMS list overridden to '$DKMS_LIST' (was '$DKMS_INPUT')" @@ -428,16 +409,7 @@ jobs: --debian-revision "$DEBIAN_REVISION" ) - # All three version fields come from derive-localversion.sh, which - # composed them from the ref and HEAD together. Nothing here appends - # to them: the SHA is already inside LOCALVERSION, and GITSHA carries - # it separately for the Debian version, which joins it differently. [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") - [[ -n "$SNAPSHOT" ]] && ARGS+=(--snapshot "$SNAPSHOT") - # Full SHA: prepare-source.sh takes the first 12 for the version - # strings and records the whole thing in the changelog, alongside the - # repository and ref that the version strings do not name. - ARGS+=(--git-sha "$KERNEL_SHA" --git-clone "$KERNEL_URL" --git-ref "$KERNEL_REF") [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") # Always passed, unlike the optional inputs above: an empty list is a diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index a3022f7e..1e30522c 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -1,20 +1,20 @@ name: daily # Daily kernel package builds. Runs at 3:00 PM PST (23:00 UTC), one parallel -# build per Daily entry in ci/build-matrix.json after suite flattening. -# Manual dispatch can run the full Daily matrix, every suite for one kernel -# variant, or one kernel variant and suite. Variant and suite are matrix values, -# so future variants need no workflow edit. +# build per Daily entry in ci/build-matrix.yaml. Manual dispatch can run the +# full Daily matrix, every suite for one kernel variant, or one kernel variant +# and suite. Variant and suite are matrix values, so future variants need no +# workflow edit. # -# Matrix schema: ci/build-matrix.json -# Each row with "type": "Daily" is expanded by ci/scripts/resolve-matrix.sh: -# the "suites" array is flattened into one job per suite. All other fields -# (kernel_variant, srcpkg, binpkg, kernel_config, dkms, debian_revision, -# git_clone, branch_or_tag, ref_strategy) are passed through to -# build-kernel-deb.yml as inputs. +# Matrix schema: ci/build-matrix.yaml +# One entry is one generated package, so ci/scripts/resolve-matrix.py selects +# the entries with type: Daily and passes them through as they stand. Every +# field (kernel_variant, suite, srcpkg, binpkg, kernel_config, dkms, +# debian_revision, git_clone, branch_or_tag, ref_strategy) becomes a +# build-kernel-deb.yml input. # -# To add a daily build target: add a row to ci/build-matrix.json with -# "type": "Daily". No workflow changes needed. +# To add a daily build target: add an entry to ci/build-matrix.yaml with +# type: Daily. No workflow changes needed. on: schedule: @@ -48,12 +48,11 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: flatten the Daily rows from ci/build-matrix.json. + # configure-matrix: select the Daily entries from ci/build-matrix.yaml. # - # Delegates to ci/scripts/resolve-matrix.sh which filters type=Daily rows - # and expands each row's "suites" array into one flat entry per suite. - # Manual dispatch selects the complete matrix, every suite for a variant, or - # one variant and suite. + # Delegates to ci/scripts/resolve-matrix.py, which validates the whole matrix + # and returns the entries with type: Daily. Manual dispatch selects the + # complete matrix, every suite for a variant, or one variant and suite. # --------------------------------------------------------------------------- configure-matrix: name: Configure build matrix @@ -80,29 +79,29 @@ jobs: case "$BUILD_SCOPE" in "Full matrix") - echo "Source: ci/build-matrix.json (full Daily matrix)" + echo "Source: ci/build-matrix.yaml (full Daily matrix)" args=(--type Daily) ;; "Selected variant (all suites)") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, all suites)" + echo "Source: ci/build-matrix.yaml (variant: $INPUT_VARIANT, all suites)" args=(--type Daily --kernel-variant "$INPUT_VARIANT") ;; "Selected variant and suite") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" - args=(--type Daily --kernel-variant "$INPUT_VARIANT" --single-suite "$INPUT_SUITE") + echo "Source: ci/build-matrix.yaml (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" + args=(--type Daily --kernel-variant "$INPUT_VARIANT" --suite "$INPUT_SUITE") ;; *) echo "ERROR: unsupported build scope: $BUILD_SCOPE" >&2 exit 1 ;; esac - MATRIX=$(ci/scripts/resolve-matrix.sh "${args[@]}") + MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}") echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" echo "Matrix entries: $(echo "$MATRIX" | jq length)" # --------------------------------------------------------------------------- - # build: one parallel job per flat matrix entry. + # build: one parallel job per matrix entry. # # Each entry calls build-kernel-deb.yml as a reusable workflow. The full # set of matrix fields (suite, srcpkg, binpkg, kernel_config, dkms, etc.) is diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 4f9e5e3a..2d2888ae 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -1,6 +1,10 @@ name: PR Build # Build-test PRs through the same pipeline the daily builds use. +# +# Packaging and CI now live on one branch, so a PR is tested entirely against +# its own commit: its ci/build-matrix.yaml defines the legs, and its debian/ +# tree and ci/scripts build them. Nothing is read from main. on: pull_request: @@ -16,7 +20,7 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: flatten the Daily rows from the PR's build-matrix.json. + # configure-matrix: select the Daily entries from the PR's build-matrix.yaml. # # The PR head, not the merge commit: a push mid-run must not change what this # run tested. A SHA also resolves for fork PRs, whose head branch does not @@ -37,12 +41,12 @@ jobs: id: set-matrix run: | set -euo pipefail - MATRIX=$(ci/scripts/resolve-matrix.sh --type Daily) + MATRIX=$(ci/scripts/resolve-matrix.py --type Daily) echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" echo "Matrix entries: $(echo "$MATRIX" | jq length)" # --------------------------------------------------------------------------- - # build: one parallel job per flat matrix entry, mirroring daily.yml. + # build: one parallel job per matrix entry, mirroring daily.yml. # # fail-fast is disabled so a broken trixie leg still leaves forky and # resolute results on the PR. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 32eddcf5..62e31eea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,14 +2,15 @@ name: release # Release kernel package builds. Triggered manually only. # -# Reads Release rows from ci/build-matrix.json, flattens suites, and builds -# each entry using the matrix-selected ref strategy and branch_or_tag. After a -# successful build, packages are promoted from the ephemeral Debusine CI -# workspace to the stable target workspace (e.g. qli) via debusine-action -# lib/release. +# Reads the Release entries from ci/build-matrix.yaml and builds each one using +# the matrix-selected ref strategy and branch_or_tag. After a successful build, +# packages are promoted from the ephemeral Debusine CI workspace to the stable +# target workspace (e.g. qli) via debusine-action lib/release. # -# To update a release ref: open a PR that changes the Release row for that -# kernel variant in ci/build-matrix.json. +# To update a release ref: open a PR that changes branch_or_tag on every +# Release entry for that kernel variant in ci/build-matrix.yaml. A variant's +# Release entries must agree on the ref, so resolve-matrix.py rejects the +# matrix if only some of them are updated. # # This workflow intentionally has no schedule trigger. Releases are always # initiated manually after validation. @@ -42,7 +43,7 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: flatten the Release rows from ci/build-matrix.json. + # configure-matrix: select the Release entries from ci/build-matrix.yaml. # --------------------------------------------------------------------------- configure-matrix: name: Configure release matrix @@ -67,25 +68,25 @@ jobs: case "$RELEASE_SCOPE" in "Selected variant (all suites)") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, all suites)" + echo "Source: ci/build-matrix.yaml (variant: $INPUT_VARIANT, all suites)" args=(--type Release --kernel-variant "$INPUT_VARIANT") ;; "Selected variant and suite") - echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" - args=(--type Release --kernel-variant "$INPUT_VARIANT" --single-suite "$INPUT_SUITE") + echo "Source: ci/build-matrix.yaml (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" + args=(--type Release --kernel-variant "$INPUT_VARIANT" --suite "$INPUT_SUITE") ;; *) echo "ERROR: unsupported release scope: $RELEASE_SCOPE" >&2 exit 1 ;; esac - MATRIX=$(ci/scripts/resolve-matrix.sh "${args[@]}") + MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}") echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" echo "Matrix entries: $(echo "$MATRIX" | jq length)" # --------------------------------------------------------------------------- - # build: one parallel job per flat Release matrix entry. + # build: one parallel job per Release matrix entry. # # Uses the ref strategy and branch_or_tag selected by the matrix. # Passes target_workspace so build-kernel-debusine.yml triggers the Debusine diff --git a/.gitignore b/.gitignore index 2583b54a..d2fc2efd 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ debian/linux-image-*.prerm # Default kernel source and build output directories kernel-source/ kernel-build/ + +# Python bytecode cache from ci/scripts/ +__pycache__/ diff --git a/ci/build-matrix.json b/ci/build-matrix.json deleted file mode 100644 index a28175d4..00000000 --- a/ci/build-matrix.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "suite_suffix_mapping": { - "trixie": "~bpo13+1", - "forky": "", - "resolute": "~26.04.1" - }, - "deliveries": [ - { - "kernel_variant": "qcom-next", - "type": "Daily", - "suites": [ - "trixie", - "forky", - "resolute" - ], - "git_clone": "https://github.com/qualcomm-linux/kernel", - "branch_or_tag": "qcom-next", - "ref_strategy": "latest_tag", - "tag_pattern": "qcom-next-*", - "srcpkg": "linux-qcom-next", - "binpkg": "linux-image-qcom-next", - "kernel_config": [], - "dkms": [ - "kgsl", - "camx", - "iris-vpu", - "audioreach" - ], - "debian_version_stub": "0qli1", - "debian_version_suffix": "~" - }, - { - "kernel_variant": "qcom-next", - "type": "Release", - "target_workspace": "qli", - "suites": [ - "trixie", - "forky" - ], - "git_clone": "https://github.com/qualcomm-linux/kernel", - "branch_or_tag": "qcom-next-7.2-20260826", - "ref_strategy": "pinned_ref", - "srcpkg": "linux-qcom-next", - "binpkg": "linux-image-qcom-next", - "kernel_config": [], - "dkms": [ - "kgsl", - "camx", - "iris-vpu", - "audioreach" - ], - "debian_version_stub": "0qli1", - "debian_version_suffix": "" - }, - { - "kernel_variant": "qcom-next-debug", - "type": "Daily", - "suites": [ - "trixie", - "forky" - ], - "git_clone": "https://github.com/qualcomm-linux/kernel", - "branch_or_tag": "qcom-next", - "ref_strategy": "latest_tag", - "tag_pattern": "qcom-next-*", - "srcpkg": "linux-qcom-next-debug", - "binpkg": "linux-image-qcom-next-debug", - "kernel_config": [ - "intree:arch/arm64/configs/qcom_debug.config", - "intree:kernel/configs/debug.config" - ], - "dkms": [ - "kgsl", - "camx", - "iris-vpu", - "audioreach" - ], - "debian_version_stub": "0qli1", - "debian_version_suffix": "~" - }, - { - "kernel_variant": "qcom-next-debug", - "type": "Release", - "target_workspace": "qli", - "suites": [ - "trixie", - "forky" - ], - "git_clone": "https://github.com/qualcomm-linux/kernel", - "branch_or_tag": "qcom-next-7.2-20260826", - "ref_strategy": "pinned_ref", - "srcpkg": "linux-qcom-next-debug", - "binpkg": "linux-image-qcom-next-debug", - "kernel_config": [ - "intree:arch/arm64/configs/qcom_debug.config", - "intree:kernel/configs/debug.config" - ], - "dkms": [ - "kgsl", - "camx", - "iris-vpu", - "audioreach" - ], - "debian_version_stub": "0qli1", - "debian_version_suffix": "" - } - ] -} diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml new file mode 100644 index 00000000..8bb0e556 --- /dev/null +++ b/ci/build-matrix.yaml @@ -0,0 +1,192 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear +# +# Kernel delivery matrix. +# +# One entry in "deliveries" is one generated package: a single kernel_variant, +# a single delivery type, and a single suite. The only list-valued fields are +# kernel_config and dkms, which name the fragments and the out-of-tree modules +# that go into that one package, so an entry needs no expansion — ci/scripts/ +# resolve-matrix.py validates the document, filters it, and hands the matching +# entries straight to the workflow matrix. +# +# Entries are written out in full rather than sharing YAML anchors, so each one +# can be read, grepped, and changed on its own. resolve-matrix.py enforces the +# consistency that duplication would otherwise put at risk: a variant's entries +# must agree on srcpkg, binpkg and kernel_config, and its entries of one +# delivery type must agree on the kernel ref they build. +# +# debian_revision is stated outright, not derived. Daily revisions end in "~" +# so they sort below the Release revision for the same suite; the rest of the +# revision places the suite where it belongs relative to the other suites (see +# the ordering discussion in README.md). No two entries may produce the same +# srcpkg at the same debian_revision. + +deliveries: + # --------------------------------------------------------------------------- + # qcom-next — the standard kernel. + # --------------------------------------------------------------------------- + - kernel_variant: qcom-next + type: Daily + suite: trixie + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1~' + + - kernel_variant: qcom-next + type: Daily + suite: forky + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~' + + - kernel_variant: qcom-next + type: Daily + suite: resolute + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~26.04.1~' + + - kernel_variant: qcom-next + type: Release + suite: trixie + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-rc7-20260821 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1' + target_workspace: qli + + - kernel_variant: qcom-next + type: Release + suite: forky + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-rc7-20260821 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1' + target_workspace: qli + + # --------------------------------------------------------------------------- + # qcom-next-debug — the same kernel ref built with the in-tree debug + # fragments, packaged separately so it installs alongside qcom-next. + # --------------------------------------------------------------------------- + - kernel_variant: qcom-next-debug + type: Daily + suite: trixie + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next-debug + binpkg: linux-image-qcom-next-debug + kernel_config: + - intree:arch/arm64/configs/qcom_debug.config + - intree:kernel/configs/debug.config + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1~' + + - kernel_variant: qcom-next-debug + type: Daily + suite: forky + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next-debug + binpkg: linux-image-qcom-next-debug + kernel_config: + - intree:arch/arm64/configs/qcom_debug.config + - intree:kernel/configs/debug.config + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~' + + - kernel_variant: qcom-next-debug + type: Release + suite: trixie + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-rc7-20260821 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next-debug + binpkg: linux-image-qcom-next-debug + kernel_config: + - intree:arch/arm64/configs/qcom_debug.config + - intree:kernel/configs/debug.config + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1' + target_workspace: qli + + - kernel_variant: qcom-next-debug + type: Release + suite: forky + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-rc7-20260821 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next-debug + binpkg: linux-image-qcom-next-debug + kernel_config: + - intree:arch/arm64/configs/qcom_debug.config + - intree:kernel/configs/debug.config + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1' + target_workspace: qli diff --git a/ci/scripts/derive-debian-revision.sh b/ci/scripts/derive-debian-revision.sh deleted file mode 100755 index 008e993e..00000000 --- a/ci/scripts/derive-debian-revision.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# SPDX-License-Identifier: BSD-3-Clause-Clear -set -euo pipefail - -# Derive the suite-specific Debian revision for one delivery leg. -# -# Formula: -# debian_revision = stub + suite_suffix_mapping[suite] + delivery_suffix -# delivery_suffix: Daily -> "~", Release -> "" -# -# This is the single implementation of the formula. It is called both by -# resolve-matrix.sh (once per flattened Daily/Release leg) and by -# build-kernel-deb.yml's direct-dispatch path (one suite, no full matrix -# context), so the derivation and its validation live in exactly one place. -# -# Usage: -# ci/scripts/derive-debian-revision.sh --stub 0qli1 --suite trixie --delivery-type Daily -# ci/scripts/derive-debian-revision.sh --stub 0qli1 --suite forky --delivery-type Release --matrix-file ci/build-matrix.json -# -# Options: -# --stub STUB Debian version stub, e.g. 0qli1. Must end in a digit: -# that digit is the packaging revision, bumped for a -# rebuild of an unchanged kernel snapshot. Must not -# end in ~ (the delivery suffix supplies any -# trailing ~). Required. -# --suite SUITE Target suite; must have an entry in -# suite_suffix_mapping. Required. -# --delivery-type TYPE Daily or Release. Required. -# --matrix-file FILE Path to the matrix JSON containing -# suite_suffix_mapping -# (default: ci/build-matrix.json relative to CWD). -# -# Output: -# Final Debian revision printed to stdout. -# -# Exit codes: -# 0 Success. -# 1 Error (invalid args, malformed or missing suite_suffix_mapping, -# unmapped suite, unsupported delivery type). - -STUB="" -SUITE="" -DELIVERY_TYPE="" -MATRIX_FILE="ci/build-matrix.json" - -usage() { - sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' - exit 1 -} - -while [[ $# -gt 0 ]]; do - case $1 in - --stub) STUB="$2"; shift 2 ;; - --suite) SUITE="$2"; shift 2 ;; - --delivery-type) DELIVERY_TYPE="$2"; shift 2 ;; - --matrix-file) MATRIX_FILE="$2"; shift 2 ;; - -h|--help) usage ;; - *) echo "ERROR: Unknown option: $1" >&2; usage ;; - esac -done - -[[ -n "$STUB" ]] || { echo "ERROR: --stub is required" >&2; exit 1; } -[[ -n "$SUITE" ]] || { echo "ERROR: --suite is required" >&2; exit 1; } -[[ -n "$DELIVERY_TYPE" ]] || { echo "ERROR: --delivery-type is required" >&2; exit 1; } -[[ "$STUB" != *"~" ]] || { echo "ERROR: --stub must not end in ~ (got '$STUB')" >&2; exit 1; } -# The trailing digit is the packaging revision: it is the only field left to -# bump when the kernel snapshot is unchanged but the packaging is rebuilt. -# suite_suffix_mapping is a per-suite constant and cannot carry it, and the -# delivery suffix is the Daily/Release marker, so a stub without a digit leaves -# a rebuild with nowhere to go. -[[ "$STUB" =~ [0-9]$ ]] || { - echo "ERROR: --stub must end in a digit, the packaging revision (got '$STUB'; use '${STUB}1')" >&2 - exit 1 -} -[[ -f "$MATRIX_FILE" ]] || { echo "ERROR: Matrix file not found: $MATRIX_FILE" >&2; exit 1; } - -jq empty "$MATRIX_FILE" 2>/dev/null \ - || { echo "ERROR: Invalid JSON in $MATRIX_FILE" >&2; exit 1; } - -mapping_errors=$(jq -r ' - .suite_suffix_mapping as $mapping | - if ($mapping | type) != "object" - then "suite_suffix_mapping is missing or not an object" - else - ( - [$mapping | to_entries[] | select(.value | type != "string") | "suite_suffix_mapping[" + .key + "] must be a string"] - + - [ - $mapping - | to_entries[] - | select((.value | type == "string") and .value != "" and (.value | test("^~") | not)) - | "suite_suffix_mapping[" + .key + "] must be empty or start with ~ (got \"" + .value + "\")" - ] - + - [ - $mapping - | to_entries - | group_by(.value) - | map(select(length > 1)) - | .[]? - | "suites " + ([.[].key] | join(", ")) + " share the same suffix \"" + .[0].value + "\"" - ] - ) | .[] - end -' "$MATRIX_FILE") - -if [[ -n "$mapping_errors" ]]; then - echo "ERROR: Invalid suite_suffix_mapping in $MATRIX_FILE:" >&2 - while IFS= read -r error; do - [[ -n "$error" ]] && echo " - $error" >&2 - done <<< "$mapping_errors" - exit 1 -fi - -SUFFIX=$(jq -r --arg suite "$SUITE" '.suite_suffix_mapping[$suite] // "__MISSING__"' "$MATRIX_FILE") -[[ "$SUFFIX" != "__MISSING__" ]] || { - echo "ERROR: no suite_suffix_mapping entry for suite '$SUITE'" >&2 - exit 1 -} - -case "$DELIVERY_TYPE" in - Daily) DELIVERY_SUFFIX="~" ;; - Release) DELIVERY_SUFFIX="" ;; - *) - echo "ERROR: --delivery-type must be Daily or Release (got '$DELIVERY_TYPE')" >&2 - exit 1 - ;; -esac - -echo "${STUB}${SUFFIX}${DELIVERY_SUFFIX}" diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py new file mode 100755 index 00000000..4f97703c --- /dev/null +++ b/ci/scripts/resolve-matrix.py @@ -0,0 +1,523 @@ +#!/usr/bin/env python3 +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear +"""Validate and select entries from the kernel delivery matrix. + +ci/build-matrix.yaml holds one entry per generated package: a single +kernel_variant, a single delivery type, and a single suite. Nothing here +expands or derives anything -- the matrix is already flat, and each entry +states its own debian_revision. This script validates the whole document, +selects the entries a caller asked for, and prints them. + +The document is validated in full on every invocation, not just the selected +entries, so a typo in a Release entry fails the daily build too rather than +lying in wait until someone runs a release. + +Usage: + ci/scripts/resolve-matrix.py --type Daily + ci/scripts/resolve-matrix.py --type Release --kernel-variant qcom-next + ci/scripts/resolve-matrix.py --type Daily --kernel-variant qcom-next --suite trixie + ci/scripts/resolve-matrix.py --type Daily --kernel-variant qcom-next \\ + --suite trixie --field debian_revision + +Options: + --type TYPE Delivery type to select (Daily or Release). + Required. + --kernel-variant VARIANT Select only this kernel variant. + --suite SUITE Select only this suite. + --field NAME Print just this field of the single selected + entry, unquoted. Errors unless exactly one + entry matches. + --matrix-file FILE Matrix path (default: ci/build-matrix.yaml + relative to CWD). + +Output: + Without --field, a compact JSON array of the selected entries, ready for a + GitHub Actions matrix `include`. kernel_config and dkms are joined into the + comma-separated strings that build-kernel-deb.yml's kernel-config and dkms + inputs, and prepare-source.sh's --kernel-config and --dkms, expect; every + other field is passed through as written. + + With --field, the named field's value alone, so a workflow step can capture + it directly. + +Exit codes: + 0 Success, at least one entry selected. + 1 Error (invalid arguments, matrix validation failure, no matching entry, + or --field matching more than one entry). +""" + +import argparse +import json +import re +import sys +from collections import defaultdict + +try: + import yaml +except ImportError: + sys.exit( + "ERROR: PyYAML is required to read the delivery matrix.\n" + " Install it with 'apt-get install python3-yaml' or 'pip install pyyaml'." + ) + +DEFAULT_MATRIX_FILE = "ci/build-matrix.yaml" + +DELIVERY_TYPES = ("Daily", "Release") +REF_STRATEGIES = ("latest_tag", "branch_tip", "pinned_ref") + +# A delivery type constrains how its kernel ref is chosen: a Daily build tracks +# something moving, a Release build is pinned to an immutable ref. +REF_STRATEGIES_FOR_TYPE = { + "Daily": ("latest_tag", "branch_tip"), + "Release": ("pinned_ref",), +} + +REQUIRED_STRING_FIELDS = ( + "kernel_variant", + "type", + "suite", + "git_clone", + "branch_or_tag", + "ref_strategy", + "srcpkg", + "binpkg", + "debian_revision", +) + +OPTIONAL_STRING_FIELDS = ( + "tag_pattern", + "target_workspace", + "localversion", + "kver_extra", + "debusine_parent_workspace", +) + +KNOWN_FIELDS = frozenset( + REQUIRED_STRING_FIELDS + OPTIONAL_STRING_FIELDS + ("kernel_config", "dkms") +) + +# Fields that identify the variant itself rather than one of its build legs. +# Every entry for a variant must agree on them, because they decide what the +# package is; the entries only differ in where it is delivered. +VARIANT_IDENTITY_FIELDS = ("srcpkg", "binpkg", "kernel_config", "dkms") + +# Fields deciding which kernel tree is built. All entries for one variant and +# delivery type build the same source, so a stale suite cannot quietly ship a +# different kernel from its siblings. +REF_FIELDS = ("git_clone", "branch_or_tag", "ref_strategy", "tag_pattern") + +NAME_RE = re.compile(r"^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$") + +# A Debian revision: no hyphen (that would start a new revision component) and +# none of the characters dpkg rejects in a version. +REVISION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.+~]*$") + +# An "intree:" entry names a fragment shipped by the kernel source, as a path +# relative to the kernel source root. A bare entry names a fragment in +# debian/config-available/, with or without its .config extension. +INTREE_PATH_RE = re.compile(r"^([A-Za-z0-9_.-]+/)*[A-Za-z0-9_.-]+\.config$") +BARE_FRAGMENT_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + +# A dkms entry is a package name stem: the build wants -dkms available, +# and generates both the Build-Depends entry and the debian/dkms-modules +# manifest from it. Same shape debian/rules enforces at prepare time, checked +# here so a typo fails before a runner is claimed rather than mid-build. +DKMS_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9+.-]*$") + + +def fragment_basename(fragment): + """Filename a fragment lands under in debian/config/, minus .config. + + Every fragment is copied into debian/config/ under its basename, so + arch/arm64/configs/hardening.config and kernel/configs/hardening.config + collide there even though the entries differ. + """ + return fragment.removeprefix("intree:").rsplit("/", 1)[-1].removesuffix(".config") + + +def check_kernel_config(entry, report): + """Validate one entry's kernel_config list.""" + fragments = entry.get("kernel_config") + if not isinstance(fragments, list): + report("kernel_config must be a list") + return + if any(not isinstance(f, str) or not f for f in fragments): + report("kernel_config must contain only non-empty strings") + return + if any("," in f for f in fragments): + report( + "kernel_config entries must not contain commas; " + "use one list element per fragment" + ) + return + if len(set(fragments)) != len(fragments): + report("kernel_config must not contain duplicates") + + for fragment in fragments: + if fragment.startswith("intree:"): + path = fragment.removeprefix("intree:") + traversal = path == ".." or path.startswith("../") or "/../" in path + if not INTREE_PATH_RE.match(path) or traversal or path.endswith("/.."): + report( + f"intree: entry '{fragment}' must be a kernel-source-relative " + "path ending in .config " + "(e.g. intree:arch/arm64/configs/qcom_debug.config)" + ) + elif not BARE_FRAGMENT_RE.match(fragment): + report( + f"kernel_config entry '{fragment}' must be a fragment name from " + "debian/config-available/ or an intree: path" + ) + + basenames = [fragment_basename(f) for f in fragments] + if len(set(basenames)) != len(basenames): + report("kernel_config entries must not resolve to the same fragment filename") + + +def check_dkms(entry, report): + """Validate one entry's dkms list. + + An empty list is meaningful: it bundles nothing, as opposed to leaving the + field out, which the matrix does not allow. + """ + modules = entry.get("dkms") + if not isinstance(modules, list): + report("dkms must be a list") + return + if any(not isinstance(m, str) or not m for m in modules): + report("dkms must contain only non-empty strings") + return + if len(set(modules)) != len(modules): + report("dkms must not contain duplicates") + + for module in modules: + if module.endswith("-dkms"): + report( + f"dkms entry '{module}' must omit the -dkms suffix " + f"(use '{module.removesuffix('-dkms')}')" + ) + elif not DKMS_NAME_RE.match(module): + report(f"dkms entry '{module}' must be a package name stem, e.g. kgsl") + + +def check_entry(entry, report): + """Validate one delivery entry in isolation.""" + for field in REQUIRED_STRING_FIELDS: + value = entry.get(field) + if not isinstance(value, str) or not value: + report(f"missing or invalid {field}") + + for field in OPTIONAL_STRING_FIELDS: + if field in entry and not isinstance(entry[field], str): + report(f"invalid {field}") + + for field in sorted(set(entry) - KNOWN_FIELDS): + report(f"unknown field {field}") + + for field in ("kernel_variant", "suite"): + value = entry.get(field) + if isinstance(value, str) and not NAME_RE.match(value): + report(f"{field} must use lowercase letters, digits, and internal hyphens") + + check_kernel_config(entry, report) + check_dkms(entry, report) + + delivery_type = entry.get("type") + if delivery_type not in DELIVERY_TYPES: + report("type must be Daily or Release") + + ref_strategy = entry.get("ref_strategy") + if ref_strategy not in REF_STRATEGIES: + report("ref_strategy must be " + ", ".join(REF_STRATEGIES)) + elif delivery_type in REF_STRATEGIES_FOR_TYPE: + allowed = REF_STRATEGIES_FOR_TYPE[delivery_type] + if ref_strategy not in allowed: + report( + f"{delivery_type} entries must use " + + " or ".join(f"ref_strategy={s}" for s in allowed) + ) + + if ref_strategy == "latest_tag": + if not entry.get("tag_pattern"): + report("missing or invalid tag_pattern") + elif "tag_pattern" in entry: + report("tag_pattern is only valid with ref_strategy=latest_tag") + + if delivery_type == "Release": + if not entry.get("target_workspace"): + report("missing or invalid target_workspace") + elif "target_workspace" in entry: + report("target_workspace is only valid for Release") + + revision = entry.get("debian_revision") + if isinstance(revision, str) and revision: + if not REVISION_RE.match(revision): + report( + f'debian_revision "{revision}" is not a valid Debian revision ' + "(letters, digits, and . + ~ only, starting with a letter or digit)" + ) + # A trailing ~ sorts a version below the same version without it, so a + # Daily always sorts below the Release it will be superseded by. + elif delivery_type == "Daily" and not revision.endswith("~"): + report(f'debian_revision "{revision}" must end in ~ for a Daily entry') + elif delivery_type == "Release" and revision.endswith("~"): + report(f'debian_revision "{revision}" must not end in ~ for a Release entry') + + +def describe(entry, index): + """Label an entry in an error message by what identifies it to a reader.""" + if not isinstance(entry, dict): + return f"entry {index}" + parts = [ + str(entry[field]) + for field in ("kernel_variant", "type", "suite") + if isinstance(entry.get(field), str) + ] + return f"entry {index} ({'/'.join(parts)})" if parts else f"entry {index}" + + +def check_consistency(deliveries, errors): + """Validate the invariants that span entries. + + Entries are written out in full, so the matrix can state a variant twice + and disagree with itself. These checks are what makes that duplication + safe to read at face value. + """ + by_leg = defaultdict(list) + by_variant = defaultdict(list) + by_variant_type = defaultdict(list) + by_package_version = defaultdict(list) + variants_by_package = defaultdict(set) + + for entry in deliveries: + if not isinstance(entry, dict): + continue + variant = entry.get("kernel_variant") + delivery_type = entry.get("type") + suite = entry.get("suite") + if not isinstance(variant, str): + continue + + by_variant[variant].append(entry) + by_variant_type[(variant, delivery_type)].append(entry) + by_leg[(variant, delivery_type, suite)].append(entry) + + for field in ("srcpkg", "binpkg"): + if isinstance(entry.get(field), str) and entry[field]: + variants_by_package[(field, entry[field])].add(variant) + + if isinstance(entry.get("srcpkg"), str) and isinstance( + entry.get("debian_revision"), str + ): + by_package_version[(entry["srcpkg"], entry["debian_revision"])].append(entry) + + for (variant, delivery_type, suite), entries in sorted( + by_leg.items(), key=lambda item: str(item[0]) + ): + if len(entries) > 1: + errors.append( + f"kernel_variant {variant} defines {len(entries)} {delivery_type} " + f"entries for suite {suite}; one entry is one generated package" + ) + + for variant, entries in sorted(by_variant.items()): + for field in VARIANT_IDENTITY_FIELDS: + values = {json.dumps(entry.get(field), sort_keys=True) for entry in entries} + if len(values) > 1: + errors.append( + f"kernel_variant {variant} must use one {field} across all its " + "entries (got " + ", ".join(sorted(values)) + ")" + ) + types = {entry.get("type") for entry in entries} + for delivery_type in DELIVERY_TYPES: + if delivery_type not in types: + errors.append( + f"kernel_variant {variant} has no {delivery_type} entry; " + "every variant must define at least one of each" + ) + + for (variant, delivery_type), entries in sorted( + by_variant_type.items(), key=lambda item: str(item[0]) + ): + for field in REF_FIELDS: + values = {entry.get(field) for entry in entries} + if len(values) > 1: + rendered = ", ".join(sorted(str(v) for v in values)) + errors.append( + f"kernel_variant {variant} must build one {field} across its " + f"{delivery_type} entries (got {rendered})" + ) + + for (field, package), variants in sorted(variants_by_package.items()): + if len(variants) > 1: + errors.append( + f"{field} {package} is shared by kernel variants " + + ", ".join(sorted(variants)) + ) + + for (srcpkg, revision), entries in sorted( + by_package_version.items(), key=lambda item: str(item[0]) + ): + if len(entries) > 1: + suites = ", ".join(sorted(str(entry.get("suite")) for entry in entries)) + errors.append( + f"srcpkg {srcpkg} is built at debian_revision {revision} for " + f"suites {suites}; each entry needs a revision of its own" + ) + + # A suite's Daily and Release differ only by the Daily's trailing ~, so the + # Daily reliably sorts below the Release that supersedes it. + for variant, entries in sorted(by_variant.items()): + revisions = { + (entry.get("type"), entry.get("suite")): entry.get("debian_revision") + for entry in entries + } + for (delivery_type, suite), revision in sorted( + revisions.items(), key=lambda item: str(item[0]) + ): + if delivery_type != "Daily" or not isinstance(revision, str): + continue + release = revisions.get(("Release", suite)) + if isinstance(release, str) and revision != release + "~": + errors.append( + f"kernel_variant {variant} suite {suite}: Daily " + f'debian_revision "{revision}" must be the Release revision ' + f'"{release}" with a trailing ~' + ) + + +def validate(deliveries): + """Return every problem found in the matrix, as a list of messages.""" + errors = [] + for index, entry in enumerate(deliveries): + if not isinstance(entry, dict): + errors.append(f"{describe(entry, index)}: delivery entries must be mappings") + continue + label = describe(entry, index) + check_entry(entry, lambda message, label=label: errors.append(f"{label}: {message}")) + check_consistency(deliveries, errors) + return errors + + +def load_matrix(path): + """Read, parse, and validate the matrix, returning its deliveries.""" + try: + with open(path, encoding="utf-8") as handle: + document = yaml.safe_load(handle) + except FileNotFoundError: + sys.exit(f"ERROR: Matrix file not found: {path}") + except OSError as error: + sys.exit(f"ERROR: Cannot read {path}: {error}") + except yaml.YAMLError as error: + sys.exit(f"ERROR: Invalid YAML in {path}: {error}") + + if not isinstance(document, dict): + sys.exit(f"ERROR: {path}: matrix root must be a mapping with a deliveries key") + + # deliveries is the whole schema. Anything else at the root is a leftover + # from an older matrix (suite_suffix_mapping, say) that would otherwise sit + # there looking authoritative while nothing read it. + unknown_root = sorted(set(document) - {"deliveries"}) + if unknown_root: + sys.exit( + f"ERROR: {path}: unknown top-level key(s) {', '.join(unknown_root)}; " + "deliveries is the only one" + ) + + deliveries = document.get("deliveries") + if not isinstance(deliveries, list): + sys.exit(f"ERROR: {path}: deliveries must be a list") + if not deliveries: + sys.exit(f"ERROR: {path}: deliveries must contain at least one entry") + + errors = validate(deliveries) + if errors: + sys.exit( + f"ERROR: Invalid kernel delivery matrix in {path}:\n" + + "\n".join(f" - {error}" for error in errors) + ) + + return deliveries + + +def select(deliveries, delivery_type, kernel_variant, suite): + """Return the entries matching the requested filters, in matrix order.""" + return [ + entry + for entry in deliveries + if entry["type"] == delivery_type + and kernel_variant in ("", entry["kernel_variant"]) + and suite in ("", entry["suite"]) + ] + + +def describe_selection(delivery_type, kernel_variant, suite): + """Render the active filters for an error message.""" + parts = [f"type={delivery_type}"] + if kernel_variant: + parts.append(f"kernel_variant={kernel_variant}") + if suite: + parts.append(f"suite={suite}") + return " ".join(parts) + + +def for_workflow(entry): + """Shape one entry the way build-kernel-deb.yml's inputs expect it.""" + return { + **entry, + "kernel_config": ",".join(entry["kernel_config"]), + "dkms": ",".join(entry["dkms"]), + } + + +def main(): + parser = argparse.ArgumentParser( + description="Validate and select entries from the kernel delivery matrix.", + epilog="See the module docstring in this file for full documentation.", + ) + parser.add_argument( + "--type", required=True, choices=DELIVERY_TYPES, help="delivery type to select" + ) + parser.add_argument( + "--kernel-variant", default="", help="select only this kernel variant" + ) + parser.add_argument("--suite", default="", help="select only this suite") + parser.add_argument( + "--field", + default="", + help="print just this field of the single selected entry", + ) + parser.add_argument( + "--matrix-file", default=DEFAULT_MATRIX_FILE, help="path to the matrix YAML" + ) + args = parser.parse_args() + + deliveries = load_matrix(args.matrix_file) + selected = select(deliveries, args.type, args.kernel_variant, args.suite) + + what = describe_selection(args.type, args.kernel_variant, args.suite) + if not selected: + sys.exit(f"ERROR: No matrix entries found for {what}") + + if not args.field: + print(json.dumps([for_workflow(entry) for entry in selected], separators=(",", ":"))) + return + + if len(selected) > 1: + sys.exit( + f"ERROR: --field {args.field} needs exactly one entry, but {what} " + f"selects {len(selected)}; narrow it with --kernel-variant and --suite" + ) + + entry = for_workflow(selected[0]) + if args.field not in entry: + sys.exit( + f"ERROR: {what} has no field {args.field}; " + "available: " + ", ".join(sorted(entry)) + ) + print(entry[args.field]) + + +if __name__ == "__main__": + main() diff --git a/ci/scripts/resolve-matrix.sh b/ci/scripts/resolve-matrix.sh deleted file mode 100755 index 4c120641..00000000 --- a/ci/scripts/resolve-matrix.sh +++ /dev/null @@ -1,400 +0,0 @@ -#!/bin/bash -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# SPDX-License-Identifier: BSD-3-Clause-Clear -set -euo pipefail - -# Resolve and flatten the kernel delivery matrix for a given delivery type. -# -# The matrix root is an object with two top-level keys: -# - "suite_suffix_mapping": a suite -> Debian suffix map shared by every -# kernel variant and delivery type (e.g. "trixie": "~bpo13+1"). -# - "deliveries": the matrix rows. Each kernel_variant owns exactly one -# Daily row and one Release row. A row declares every input needed by -# that delivery, including a debian_version_stub. Three fields are -# list-valued: suites, which is expanded into isolated legs; -# kernel_config, which is one config fragment per element; and dkms, -# which is one out-of-tree module per element. A fragment is -# either a bare name from debian/config-available/ or an "intree:" entry -# naming a path relative to the kernel source root -# (e.g. intree:arch/arm64/configs/qcom_debug.config). A dkms entry is the -# stem of a -dkms package, e.g. "kgsl". -# -# Each flattened leg's final debian_revision is derived from -# debian_version_stub, suite_suffix_mapping[suite], and the delivery type via -# ci/scripts/derive-debian-revision.sh, so the formula has exactly one -# implementation shared with build-kernel-deb.yml's direct-dispatch path. Each -# row also carries debian_version_suffix ("~" for Daily, "" for Release) as a -# visible, validated record of that same delivery-type mapping; it is checked -# against the row's type but never fed into derivation, so a copy/paste error -# here fails fast instead of silently drifting from the formula's single -# implementation. -# -# Usage: -# ci/scripts/resolve-matrix.sh --type Daily -# ci/scripts/resolve-matrix.sh --type Release -# ci/scripts/resolve-matrix.sh --type Daily --single-suite trixie -# ci/scripts/resolve-matrix.sh --type Daily --kernel-variant qcom-next -# ci/scripts/resolve-matrix.sh --type Daily --matrix-file path/to/matrix.json -# -# Options: -# --type TYPE Delivery type to filter (Daily or Release). -# Required. -# --single-suite SUITE Emit only entries for this suite. -# --kernel-variant VARIANT Emit only entries for this kernel variant. -# --matrix-file FILE Path to the matrix JSON file -# (default: ci/build-matrix.json relative to CWD). -# -# Output: -# Compact JSON array to stdout. Every entry has a single suite, the -# kernel_variant that scopes its artifacts, Debusine workspace, and logs, -# and a suite-specific debian_revision (debian_version_stub and -# debian_version_suffix are consumed and removed). kernel_config and dkms are -# joined into the comma-separated strings that build-kernel-deb.yml's -# kernel-config and dkms inputs — and in turn prepare-source.sh's -# --kernel-config and --dkms — expect. -# -# Exit codes: -# 0 Success, at least one entry emitted. -# 1 Error (invalid arguments, matrix validation failure, no matching -# entry, revision derivation failure). - -TYPE="" -SINGLE_SUITE="" -KERNEL_VARIANT="" -MATRIX_FILE="ci/build-matrix.json" - -usage() { - sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' - exit 1 -} - -while [[ $# -gt 0 ]]; do - case $1 in - --type) TYPE="$2"; shift 2 ;; - --single-suite) SINGLE_SUITE="$2"; shift 2 ;; - --kernel-variant) KERNEL_VARIANT="$2"; shift 2 ;; - --matrix-file) MATRIX_FILE="$2"; shift 2 ;; - -h|--help) usage ;; - *) echo "ERROR: Unknown option: $1" >&2; usage ;; - esac -done - -[[ "$TYPE" == "Daily" || "$TYPE" == "Release" ]] || { - echo "ERROR: --type must be Daily or Release" >&2 - exit 1 -} -[[ -f "$MATRIX_FILE" ]] || { echo "ERROR: Matrix file not found: $MATRIX_FILE" >&2; exit 1; } - -jq empty "$MATRIX_FILE" 2>/dev/null \ - || { echo "ERROR: Invalid JSON in $MATRIX_FILE" >&2; exit 1; } - -validation_errors=$(jq -r ' - def required_string($field): - if (has($field) and (.[$field] | type == "string") and (.[$field] | length > 0)) - then empty - else "missing or invalid " + $field - end; - - def optional_string($field): - if (has($field) | not) or (.[$field] | type == "string") - then empty - else "invalid " + $field - end; - - def variant_name_valid: - if (.kernel_variant | type) != "string" - then empty - elif (.kernel_variant | test("^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$")) - then empty - else "kernel_variant must use lowercase letters, digits, and internal hyphens" - end; - - # An "intree:" entry names a fragment shipped by the kernel source, as a - # path relative to the kernel source root (e.g. - # intree:arch/arm64/configs/qcom_debug.config). Reject absolute paths, - # ".." traversal, and anything not ending in .config so a typo fails here - # rather than deep inside prepare-source.sh. - def intree_path_valid: - .[7:] as $path | - ($path | test("^([A-Za-z0-9_.-]+/)*[A-Za-z0-9_.-]+\\.config$")) - and (($path | test("(^|/)\\.\\.(/|$)")) | not); - - # Every fragment lands in debian/config/ under its basename, so two entries - # sharing one basename (e.g. arch/arm64/configs/hardening.config and - # kernel/configs/hardening.config) collide there. - def fragment_basename: - sub("^intree:"; "") | sub("^.*/"; "") | sub("\\.config$"; ""); - - def kernel_config_valid: - if (.kernel_config | type) != "array" - then "kernel_config must be an array" - elif any(.kernel_config[]; type != "string" or length == 0) - then "kernel_config must contain only non-empty strings" - elif any(.kernel_config[]; test(",")) - then "kernel_config entries must not contain commas; use one array element per fragment" - elif ([.kernel_config[]] | unique | length) != (.kernel_config | length) - then "kernel_config must not contain duplicates" - elif any(.kernel_config[]; startswith("intree:") and (intree_path_valid | not)) - then "intree: entries must be a kernel-source-relative path ending in .config (e.g. intree:arch/arm64/configs/qcom_debug.config)" - elif ([.kernel_config[] | fragment_basename] | unique | length) != (.kernel_config | length) - then "kernel_config entries must not resolve to the same fragment filename" - else empty - end; - - # Out-of-tree modules to build and bundle, each named as the stem of its - # -dkms package. The same shape rules debian/rules prepare enforces, - # applied here so a typo fails before any build job starts rather than in - # the middle of one. - def dkms_valid: - if (.dkms | type) != "array" - then "dkms must be an array" - elif any(.dkms[]; type != "string" or length == 0) - then "dkms must contain only non-empty strings" - elif any(.dkms[]; test(",")) - then "dkms entries must not contain commas; use one array element per module" - elif any(.dkms[]; endswith("-dkms")) - then "dkms entries must omit the -dkms suffix (e.g. kgsl, not kgsl-dkms)" - elif any(.dkms[]; test("^[a-z0-9][a-z0-9+.-]*$") | not) - then "dkms entries must be package name stems (lowercase letters, digits, + . -)" - elif ([.dkms[]] | unique | length) != (.dkms | length) - then "dkms must not contain duplicates" - else empty - end; - - def suites_valid: - if (.suites | type) != "array" or (.suites | length) == 0 - then "suites must be a non-empty array" - elif any(.suites[]; type != "string" or length == 0) - then "suites must contain only non-empty strings" - elif any(.suites[]; test("^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$") | not) - then "suites must use lowercase letters, digits, and internal hyphens" - elif ([.suites[]] | unique | length) != (.suites | length) - then "suites must not contain duplicates" - else empty - end; - - def row_errors($index): - if type != "object" - then "row " + ($index | tostring) + ": matrix entries must be objects" - else - . as $row | - [ - required_string("kernel_variant"), - required_string("type"), - required_string("git_clone"), - required_string("branch_or_tag"), - required_string("ref_strategy"), - required_string("srcpkg"), - required_string("binpkg"), - required_string("debian_version_stub"), - optional_string("debusine_parent_workspace"), - optional_string("localversion"), - optional_string("kver_extra"), - variant_name_valid, - suites_valid, - kernel_config_valid, - dkms_valid, - if (.debian_version_stub | type) == "string" and (.debian_version_stub | test("~$")) - then "debian_version_stub must not end in ~" - else empty end, - if (has("debian_version_suffix") | not) or (.debian_version_suffix | type) != "string" - then "missing or invalid debian_version_suffix" - elif .type == "Daily" and .debian_version_suffix != "~" - then "debian_version_suffix must be \"~\" for Daily rows (got \"" + (.debian_version_suffix | tostring) + "\")" - elif .type == "Release" and .debian_version_suffix != "" - then "debian_version_suffix must be \"\" for Release rows (got \"" + (.debian_version_suffix | tostring) + "\")" - else empty end, - if (.type == "Daily" or .type == "Release") - then empty else "type must be Daily or Release" end, - if (.ref_strategy == "latest_tag" or .ref_strategy == "branch_tip" or .ref_strategy == "pinned_ref") - then empty else "ref_strategy must be latest_tag, branch_tip, or pinned_ref" end, - if .type == "Daily" and (.ref_strategy != "latest_tag" and .ref_strategy != "branch_tip") - then "Daily rows must use ref_strategy=latest_tag or ref_strategy=branch_tip" - elif .type == "Release" and .ref_strategy != "pinned_ref" - then "Release rows must use ref_strategy=pinned_ref" - else empty - end, - if .ref_strategy == "latest_tag" - then required_string("tag_pattern") - elif has("tag_pattern") - then "tag_pattern is only valid with ref_strategy=latest_tag" - else empty - end, - if .type == "Release" - then required_string("target_workspace") - elif has("target_workspace") - then "target_workspace is only valid for Release" - else empty - end - ] | .[] | "row " + ($index | tostring) + " (" + (($row.kernel_variant // "unknown") | tostring) + "): " + . - end; - - if type != "object" - then "matrix root must be an object with suite_suffix_mapping and deliveries" - elif (.deliveries | type) != "array" - then "deliveries must be an array" - elif (.deliveries | length) == 0 - then "deliveries must contain at least one row" - elif (.suite_suffix_mapping | type) != "object" - then "suite_suffix_mapping is missing or not an object" - else - .deliveries as $matrix | - .suite_suffix_mapping as $mapping | - ( - [range(0; ($matrix | length)) as $index | $matrix[$index] | row_errors($index)] - + - [ - [$matrix[] | select(type == "object")] - | group_by(.kernel_variant) - | .[] - | . as $rows - | (($rows[0].kernel_variant // "unknown") | tostring) as $variant - | ([ $rows[].type ] | sort) as $types - | ([ $rows[].srcpkg ] | unique) as $srcpkgs - | ([ $rows[].binpkg ] | unique) as $binpkgs - | ([ $rows[].debian_version_stub ] | unique) as $stubs - | if ($rows | length) != 2 - then "kernel_variant " + $variant + " must define exactly one Daily row and one Release row" - elif $types != ["Daily", "Release"] - then "kernel_variant " + $variant + " must define exactly one Daily row and one Release row" - elif ($srcpkgs | length) != 1 - then "kernel_variant " + $variant + " must use one srcpkg across its Daily and Release rows" - elif ($binpkgs | length) != 1 - then "kernel_variant " + $variant + " must use one binpkg across its Daily and Release rows" - elif ($stubs | length) != 1 - then "kernel_variant " + $variant + " must use one debian_version_stub across its Daily and Release rows" - else empty - end - ] - + - [ - [ - $matrix[] - | select(type == "object") - | select((.kernel_variant | type) == "string") - | select((.srcpkg | type) == "string" and (.srcpkg | length) > 0) - | {package: .srcpkg, kernel_variant: .kernel_variant} - ] - | group_by(.package)[] - | ([.[].kernel_variant] | unique) as $variants - | select($variants | length > 1) - | "srcpkg " + .[0].package + " is shared by kernel variants " + ($variants | join(", ")) - ] - + - [ - [ - $matrix[] - | select(type == "object") - | select((.kernel_variant | type) == "string") - | select((.binpkg | type) == "string" and (.binpkg | length) > 0) - | {package: .binpkg, kernel_variant: .kernel_variant} - ] - | group_by(.package)[] - | ([.[].kernel_variant] | unique) as $variants - | select($variants | length > 1) - | "binpkg " + .[0].package + " is shared by kernel variants " + ($variants | join(", ")) - ] - + - [ - $mapping | to_entries[] | select(.value | type != "string") - | "suite_suffix_mapping[" + .key + "] must be a string" - ] - + - [ - $mapping - | to_entries[] - | select((.value | type == "string") and .value != "" and (.value | test("^~") | not)) - | "suite_suffix_mapping[" + .key + "] must be empty or start with ~ (got \"" + .value + "\")" - ] - + - [ - $mapping - | to_entries - | group_by(.value) - | map(select(length > 1)) - | .[]? - | "suites " + ([.[].key] | join(", ")) + " share the same suffix \"" + .[0].value + "\"" - ] - + - [ - [$matrix[] | select(type == "object") | select((.suites | type) == "array") | .suites[]] - | unique - | .[] as $suite - | select(($mapping | has($suite)) | not) - | "suite " + $suite + " has no suite_suffix_mapping entry" - ] - ) | .[] - end -' "$MATRIX_FILE") - -if [[ -n "$validation_errors" ]]; then - echo "ERROR: Invalid kernel delivery matrix:" >&2 - while IFS= read -r error; do - [[ -n "$error" ]] && echo " - $error" >&2 - done <<< "$validation_errors" - exit 1 -fi - -result=$(jq -c \ - --arg type "$TYPE" \ - --arg single_suite "$SINGLE_SUITE" \ - --arg kernel_variant "$KERNEL_VARIANT" ' - [ - .deliveries[] - | select(.type == $type) - | select($kernel_variant == "" or .kernel_variant == $kernel_variant) - | . as $row - | ( - if $single_suite == "" - then .suites - elif (.suites | index($single_suite)) != null - then [$single_suite] - else [] - end - )[] as $suite - | $row - | del(.suites) - | . + { - "suite": $suite, - "kernel_config": ($row.kernel_config | join(",")), - "dkms": ($row.dkms | join(",")) - } - ] - | if length == 0 - then error( - "no matrix entries found for type=" + $type - + (if $kernel_variant != "" then " kernel_variant=" + $kernel_variant else "" end) - + (if $single_suite != "" then " suite=" + $single_suite else "" end) - ) - else . - end - ' "$MATRIX_FILE") || { - echo "ERROR: Matrix resolution failed for type=$TYPE${KERNEL_VARIANT:+ kernel_variant=$KERNEL_VARIANT}${SINGLE_SUITE:+ suite=$SINGLE_SUITE}" >&2 - exit 1 -} - -# Derive each leg's final debian_revision from debian_version_stub, -# suite_suffix_mapping, and its delivery type. derive-debian-revision.sh is -# the single implementation of the formula; build-kernel-deb.yml's direct -# dispatch path calls the same script for the one-suite, no-matrix case. -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -final="[]" -while IFS= read -r leg; do - suite=$(jq -r '.suite' <<< "$leg") - stub=$(jq -r '.debian_version_stub' <<< "$leg") - delivery_type=$(jq -r '.type' <<< "$leg") - variant=$(jq -r '.kernel_variant' <<< "$leg") - - revision=$("$script_dir/derive-debian-revision.sh" \ - --stub "$stub" --suite "$suite" --delivery-type "$delivery_type" \ - --matrix-file "$MATRIX_FILE") || { - echo "ERROR: Failed to derive Debian revision for kernel_variant=$variant suite=$suite type=$delivery_type" >&2 - exit 1 - } - - leg=$(jq -c --arg rev "$revision" '(. + {debian_revision: $rev}) | del(.debian_version_stub, .debian_version_suffix)' <<< "$leg") - final=$(jq -c --argjson leg "$leg" '. + [$leg]' <<< "$final") -done < <(jq -c '.[]' <<< "$result") - -echo "$final" From 4fcfa3b38dc87869f241d1f4dfbaf9223b85358d Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 13/71] docs: describe the flattened delivery matrix Bring the READMEs onto the schema the previous commit introduced. The Matrix Model section documented suites, debian_version_stub and debian_version_suffix as fields and derive-debian-revision.sh as the single implementation of the revision formula, none of which exist now. Replace the field table with the flattened one, set out the invariants the resolver enforces across entries, and rewrite the maintenance instructions: adding a variant is now one entry per package it should produce rather than exactly two rows, and adding a suite no longer starts with a suite_suffix_mapping entry. The ordering discussion keeps its Daily-sorts-below-Release argument, which the trailing ~ still carries, but now attributes it to the revisions themselves rather than to a mapping. The example matrix becomes YAML and shows one Daily and one Release entry for the same suite, which is what the pairing rule now means. debian/README.md's pointer to the dkms field follows the file to its new name; that field is still unimplemented and is wired up separately. Signed-off-by: Christopher Obbard --- README.md | 249 +++++++++++++++++++++++++---------------------- debian/README.md | 2 +- 2 files changed, 131 insertions(+), 120 deletions(-) diff --git a/README.md b/README.md index 294e6f48..6086a544 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ into LOCALVERSION, so each produces a distinct kernel release distinct versioned image package that can be installed alongside the other. See [docs/version.md](docs/version.md) for how the version strings are composed. -`ci/build-matrix.json` is the source of truth; this table is a summary. +`ci/build-matrix.yaml` is the source of truth; this table is a summary. Two entry points use the same reusable build pipeline: @@ -41,56 +41,53 @@ Two entry points use the same reusable build pipeline: - **Release** uses a pinned matrix ref and promotes successful Debian packages to the selected production Debusine workspace. -The final Production matrix is conceptually: - -```json -{ - "suite_suffix_mapping": { - "trixie": "~bpo13+1", - "forky": "", - "resolute": "~26.04.1" - }, - "deliveries": [ - { - "kernel_variant": "qcom-next", - "type": "Daily", - "suites": ["trixie", "forky", "resolute"], - "git_clone": "https://github.com/qualcomm-linux/kernel", - "branch_or_tag": "qcom-next", - "ref_strategy": "latest_tag", - "tag_pattern": "qcom-next-*", - "srcpkg": "linux-qcom-next", - "binpkg": "linux-image-qcom-next", - "kernel_config": [], - "dkms": ["kgsl", "camx", "iris-vpu"], - "debian_version_stub": "0qli1", - "debian_version_suffix": "~" - }, - { - "kernel_variant": "qcom-next", - "type": "Release", - "suites": ["trixie", "forky"], - "git_clone": "https://github.com/qualcomm-linux/kernel", - "branch_or_tag": "", - "ref_strategy": "pinned_ref", - "srcpkg": "linux-qcom-next", - "binpkg": "linux-image-qcom-next", - "kernel_config": [], - "dkms": ["kgsl", "camx", "iris-vpu"], - "debian_version_stub": "0qli1", - "debian_version_suffix": "", - "target_workspace": "qli" - } - ] -} +One entry in `deliveries` is one generated package: a single `kernel_variant`, +a single `type`, and a single `suite`. Nothing is expanded or derived at +resolve time, so what an entry says is what gets built: + +```yaml +deliveries: + - kernel_variant: qcom-next + type: Daily + suite: trixie + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1~' + + - kernel_variant: qcom-next + type: Release + suite: trixie + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: + ref_strategy: pinned_ref + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1' + target_workspace: qli ``` -`suite_suffix_mapping` is matrix-wide policy, not duplicated per row: every -suite referenced by any row's `suites` must have an entry here, and every -delivery for a variant derives its final `debian_revision` as -`debian_version_stub + suite_suffix_mapping[suite] + delivery_suffix`, where -`delivery_suffix` is `~` for Daily and empty for Release. For the values -above: +Entries are written out in full rather than sharing YAML anchors, so each one +can be read, grepped, and changed on its own. `resolve-matrix.py` enforces the +consistency that duplication would otherwise put at risk — see +[Matrix Model](#matrix-model). + +Each entry states its `debian_revision` outright. The configured values are: | Suite | Daily | Release | | --- | --- | --- | @@ -99,19 +96,21 @@ above: | Resolute | `0qli1~26.04.1~` | (not a configured Release suite) | `~` always sorts below the same prefix without it in Debian version -ordering, so Daily always sorts below Release for the same suite and stub. -Ordering across *different* suites depends entirely on the configured -suffixes: with the mapping above, Resolute < Trixie < Forky for the same +ordering, so a suite's Daily always sorts below its Release; `resolve-matrix.py` +requires a Daily revision to be exactly its Release revision plus a trailing +`~`. Ordering across *different* suites depends entirely on the configured +revisions: with the values above, Resolute < Trixie < Forky for the same delivery type, matching a Debian-backports-then-unstable promotion chain. This is a deliberate ordering policy, not an automatic guarantee — adding a -suite means choosing a suffix that sorts where that suite belongs relative to -the others. One nuance to be aware of: because Forky's suffix is empty, its -Daily revision ends immediately after the trailing `~`, so Trixie Daily does -not sort below Forky Daily even though Trixie Release sorts below Forky -Release. This does not affect the supported Release-to-Release upgrade path. +suite means choosing a revision that sorts where that suite belongs relative to +the others. One nuance to be aware of: because Forky's revision carries no +suite component, its Daily revision ends immediately after the trailing `~`, so +Trixie Daily does not sort below Forky Daily even though Trixie Release sorts +below Forky Release. This does not affect the supported Release-to-Release +upgrade path. -`ci/build-matrix.json` is the authoritative configuration. Adding a kernel -variant is a two-row matrix change, not a workflow redesign. +`ci/build-matrix.yaml` is the authoritative configuration. Adding a kernel +variant is a matrix change, not a workflow redesign. ## Workflows @@ -165,29 +164,26 @@ production release controls. ## Matrix Model -`ci/build-matrix.json` is an object with two top-level keys: `deliveries` -(the matrix rows) and `suite_suffix_mapping` (matrix-wide Debian suffix -policy, shared by every variant and delivery type). `ci/scripts/resolve-matrix.sh` -validates the document, requires each `kernel_variant` to have exactly one -`Daily` and one `Release` row in `deliveries`, filters by delivery type, and -flattens each `suites` array into independent suite legs. Each leg carries -its own values for: +`ci/build-matrix.yaml` is a mapping with exactly one top-level key, +`deliveries`. One entry in it is one generated package, so there is no +expansion step: `ci/scripts/resolve-matrix.py` validates the whole document, +selects the entries matching the requested type, variant, and suite, and hands +them to the workflow matrix as they stand. Each entry carries: | Field | Purpose | | --- | --- | | `kernel_variant` | Stable identifier for a separately packaged kernel variant. Lowercase letters, digits, and internal hyphens only. | | `type` | `Daily` or `Release`. | -| `suites` | Suites to flatten into individual build legs. Each must have a `suite_suffix_mapping` entry. | +| `suite` | The one suite this entry builds for. | | `git_clone` | Kernel source repository. | | `branch_or_tag` | Source branch or pinned tag, according to `ref_strategy`. | | `ref_strategy` | `latest_tag`, `branch_tip`, or `pinned_ref`. | | `tag_pattern` | Required only for `latest_tag`; matching tags must end in `-YYYYMMDD`, which determines newest-first ordering. | | `srcpkg` | Debian source package name. | | `binpkg` | Kernel image metapackage name. | -| `kernel_config` | Extra fragments applied on top of `debian/config-available/`, all of which is applied to every build, one per array element. A bare name selects `debian/config-available/.config`; an `intree:` entry names a fragment shipped by the kernel source, as a path relative to the kernel source root (e.g. `intree:arch/arm64/configs/qcom_debug.config`), so it stays versioned with the kernel it targets. Empty for variants that need nothing beyond `config-available/`; today it carries only `intree:` fragments. `resolve-matrix.sh` joins it into the comma-separated `kernel-config` workflow input. | -| `dkms` | Out-of-tree DKMS modules built against this kernel and bundled into its `linux-image` package, one per array element, each named as the stem of its `-dkms` package (e.g. `kgsl`). Empty bundles nothing. A listed module is a presence contract: a build fails rather than shipping an image without it. `resolve-matrix.sh` joins it into the comma-separated `dkms` workflow input. | -| `debian_version_stub` | Base Debian revision, shared by a variant's Daily and Release rows. Must not end in `~`; the suite suffix is derived, not stored here. | -| `debian_version_suffix` | `~` for Daily rows, empty for Release rows. Documents the delivery-type half of the revision formula on the row itself; `resolve-matrix.sh` rejects a row where this disagrees with `type`, but derivation always computes this suffix from `type`, never reads this field. | +| `kernel_config` | Extra fragments applied on top of `debian/config-available/`, all of which is applied to every build, one per list element. A bare name selects `debian/config-available/.config`; an `intree:` entry names a fragment shipped by the kernel source, as a path relative to the kernel source root (e.g. `intree:arch/arm64/configs/qcom_debug.config`), so it stays versioned with the kernel it targets. Empty for variants that need nothing beyond `config-available/`; today it carries only `intree:` fragments. `resolve-matrix.py` joins it into the comma-separated `kernel-config` workflow input. | +| `dkms` | Out-of-tree DKMS modules built and bundled into the image package, one per list element, each named without the `-dkms` suffix (e.g. `kgsl`). An empty list bundles nothing. A listed module is a presence contract: a build fails rather than shipping an image without it. `resolve-matrix.py` joins it into the comma-separated `dkms` workflow input, which reaches `prepare-source.sh --dkms`; see [debian/README.md](debian/README.md) for what the packaging does with it. | +| `debian_revision` | The Debian revision this package is built at, stated outright. Daily revisions end in `~`; Release revisions do not. | | `localversion`, `kver_extra` | Optional version overrides forwarded to packaging. | | `debusine_parent_workspace` | Optional parent workspace override for the variant's CI child workspaces. | | `target_workspace` | Debusine destination for Release entries only. | @@ -195,32 +191,45 @@ its own values for: `dkms` currently has one exception the matrix cannot express: `build-kernel-deb.yml` replaces the resolved list with `kgsl` on Ubuntu-family legs, so `camx` and `iris-vpu` are bundled on Debian suites only. That override is temporary and goes -away once the matrix gains per-suite `dkms` lists. - -`target_workspace` is required for `Release` and rejected for `Daily`. -`tag_pattern` is required for `latest_tag` and rejected for other strategies. -The resolver selects the most recent trailing `YYYYMMDD` date, and rejects -duplicate suites and malformed variant identifiers before any build jobs -start. It also rejects a matrix where any configured suite has no -`suite_suffix_mapping` entry, where two suites share the same suffix, where a -suffix is non-empty and doesn't start with `~`, where a variant's Daily -and Release rows disagree on `debian_version_stub`, or where a row's -`debian_version_suffix` doesn't match what its `type` implies — all before -any build job starts. - -Each flattened leg's final `debian_revision` is derived by -`ci/scripts/derive-debian-revision.sh` from `debian_version_stub`, -`suite_suffix_mapping[suite]`, and the delivery type -(`stub + suffix + "~"` for Daily, `stub + suffix` for Release). This script is -the single implementation of the formula: `resolve-matrix.sh` calls it once -per flattened leg, and `build-kernel-deb.yml`'s direct-dispatch path (which -has no full-matrix context) calls the same script for the one suite it was -given. - -Each leg has a distinct prepared-source artifact, Debusine child workspace, and -S3 path keyed by `kernel_variant + suite`. This prevents two variants that both -build, for example, `trixie` from consuming or publishing each other's inputs -or outputs. +away once the per-suite `dkms` lists the flattened matrix now allows are used. + +`resolve-matrix.py` rejects the matrix — before any build job starts — where an +entry has an unknown field or a missing required one, a malformed variant or +suite identifier, a `ref_strategy` its `type` does not allow (`Daily` must +track something moving, `Release` must be pinned), a `tag_pattern` without +`latest_tag`, a `target_workspace` on a `Daily` entry or none on a `Release` +one, a `kernel_config` fragment that escapes the kernel source root or collides +with another fragment's filename, a `dkms` entry that is not a package name +stem or repeats, or a `debian_revision` that is not a valid Debian revision or +carries the wrong trailing `~` for its `type`. + +Because entries are written out in full, the resolver also checks the +invariants that span them, which is what makes the duplication safe to read at +face value: + +- No two entries share a `kernel_variant`, `type` and `suite` — one entry is + one generated package. +- A variant's entries agree on `srcpkg`, `binpkg`, `kernel_config` and `dkms`; + those decide what the package *is*, and the entries differ only in where it + goes. +- A variant's entries of one `type` agree on `git_clone`, `branch_or_tag`, + `ref_strategy` and `tag_pattern`, so a forgotten suite cannot quietly ship a + different kernel from its siblings after a release ref bump. +- Every variant defines at least one `Daily` and one `Release` entry. +- No `srcpkg` or `binpkg` is shared between variants, and no two entries build + the same `srcpkg` at the same `debian_revision`. +- Where a suite has both, its `Daily` revision is its `Release` revision plus a + trailing `~`. + +`build-kernel-deb.yml`'s direct-dispatch path has no matrix context of its own, +so when its `debian-revision` input is empty it looks up the `Daily` entry for +the variant and suite it was given (`resolve-matrix.py --field +debian_revision`) and builds at the revision the daily build would have used. + +Each entry has a distinct prepared-source artifact, Debusine child workspace, +and S3 path keyed by `kernel_variant + suite`. This prevents two variants that +both build, for example, `trixie` from consuming or publishing each other's +inputs or outputs. Daily S3 outputs use these layouts, where `` is `-`: @@ -237,10 +246,9 @@ Supporting scripts keep workflow YAML small and testable: | Script | Responsibility | | --- | --- | -| `ci/scripts/resolve-matrix.sh` | Validates and flattens matrix rows. | +| `ci/scripts/resolve-matrix.py` | Validates the delivery matrix and selects the entries to build. Needs PyYAML (`python3-yaml`). | | `ci/scripts/resolve-kernel-ref.sh` | Resolves a matrix-selected dated tag or validates a direct ref. | | `ci/scripts/derive-localversion.sh` | Derives the version fields from the variant, resolved kernel ref and HEAD, printing `LOCALVERSION=`, `SNAPSHOT=` and `GITSHA=` lines. `SNAPSHOT` is the dated component of the Debian version: the tag's date, or the HEAD commit date for a branch-tip build. Scheme and rationale: [docs/version.md](docs/version.md). | -| `ci/scripts/derive-debian-revision.sh` | Derives the final suite-specific `debian_revision` from `debian_version_stub`, `suite_suffix_mapping`, and delivery type. | ## Architecture @@ -451,7 +459,8 @@ The available inputs are: | `srcpkg` | `linux-qcom-next` | Advanced source package identity override. | | `binpkg` | `linux-image-qcom-next` | Advanced image metapackage identity override. | | `kernel-config` | Empty | Advanced extra fragments applied on top of all of `debian/config-available/`, e.g. `intree:arch/arm64/configs/qcom_debug.config`. | -| `debian-version-stub` | `0qli1` | Advanced Debian version stub. The selected suite's mapped suffix and a Daily-style trailing `~` are applied automatically; direct builds always use Daily semantics since they are build-only and non-promoting. | +| `dkms` | Empty | Advanced comma-separated DKMS modules bundled into the image package, each without the `-dkms` suffix. Empty bundles none. | +| `debian-revision` | The matrix Daily revision | Advanced Debian revision override. Left empty, the build takes the `debian_revision` of the matrix's `Daily` entry for the selected variant and suite; direct builds always use the Daily entry since they are build-only and non-promoting. | | `localversion` | Auto-derived | Advanced explicit `LOCALVERSION` override. | | `kver-extra` | Empty | Advanced kernel-release suffix. | | `debug-build` | `false` | Advanced debug configuration toggle. | @@ -487,37 +496,39 @@ and keeps production approval controls in the workflow path. To add a kernel variant: -1. Add exactly two rows to `deliveries` with the same `kernel_variant`: one - `Daily` and one `Release`. -2. Define all package identity, source/ref strategy, configuration, - `debian_version_stub`, and suite values in both rows. Do not rely on - another variant's values. `srcpkg`, `binpkg`, and `debian_version_stub` - must remain identical across the pair. Set `debian_version_suffix` to `~` - on the Daily row and `""` on the Release row; `resolve-matrix.sh` rejects - the pair if either disagrees with its row's `type`. +1. Add one entry to `deliveries` per package the variant should produce: one + per Daily suite and one per Release suite, each spelling out all of its own + fields. Do not rely on another variant's values. +2. Keep `srcpkg`, `binpkg` and `kernel_config` identical across every entry for + the variant, and the ref fields identical across its entries of one `type`. + `resolve-matrix.py` rejects the matrix if they drift apart. 3. Use `latest_tag` with a dated tag glob or `branch_tip` for Daily. Use `pinned_ref` for Release, and update that ref through a reviewed PR. -4. Give the variant distinct `srcpkg` and `binpkg` values. Set the Release - `target_workspace` explicitly. -5. Confirm suite-family routing: Debian suites use Debusine; Ubuntu suites use +4. Give the variant distinct `srcpkg` and `binpkg` values. Set + `target_workspace` on each Release entry. +5. Give each entry a `debian_revision`: the Daily one is the Release one for + the same suite plus a trailing `~`. +6. Confirm suite-family routing: Debian suites use Debusine; Ubuntu suites use the Docker path. -6. Run a filtered Daily validation for the new variant, then its full Daily and +7. Run a filtered Daily validation for the new variant, then its full Daily and Release flows. To add a new suite (for an existing or new variant): -1. Add an entry for it to the shared top-level `suite_suffix_mapping`, empty - or starting with `~`, and distinct from every other suite's suffix. -2. Add the suite to the `suites` array of the relevant Daily and/or Release - rows. `resolve-matrix.sh` rejects any configured suite with no mapping - entry before any build job starts. -3. Choose the suffix so the suite sorts where it belongs relative to the +1. Add one entry per delivery type the suite should get, copying the variant's + existing entry for that type and changing `suite` and `debian_revision`. +2. Choose the revision so the suite sorts where it belongs relative to the others for the same delivery type (see the ordering discussion in - [Overview](#overview)). + [Overview](#overview)), and so it does not collide with another entry + building the same `srcpkg`. No workflow dispatch choices need to be updated: manual Daily and Release inputs accept matrix-defined variant and suite strings. +Run `ci/scripts/resolve-matrix.py --type Daily` and `--type Release` locally to +validate a matrix change before pushing it; both validate the whole document, +so either one catches a mistake in the other's entries. + ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) for branch, review, and DCO diff --git a/debian/README.md b/debian/README.md index bdc1e2e6..a3a77af7 100644 --- a/debian/README.md +++ b/debian/README.md @@ -394,7 +394,7 @@ A listed module is a presence contract — if it fails to build, or its `BUILD_EXCLUSIVE` gates exclude this kernel, the package build fails rather than shipping a kernel image without it. To stop bundling a module, drop it from the list its build was given (for CI builds, the `dkms` field in -`ci/build-matrix.json`). +`ci/build-matrix.yaml`). ### Standalone developer use From 91a9c07482b23ff33214d540eedeffe630e25448 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 14/71] ci: move the Ubuntu DKMS exception into the delivery matrix camx-dkms and iris-vpu-dkms are not packaged for Ubuntu, so build-kernel-deb.yml overwrote the resolved DKMS list with kgsl on Ubuntu-family legs. That was delivery policy sitting in workflow YAML, put there because a matrix row spanned several suites and could not say anything about one of them. A row no longer does: one entry is one generated package for one suite, so the resolute entry can carry its own dkms list and does. Revert the override, along with the "needs: resolve" and SUITE_FAMILY plumbing that fed it, and pass the resolved list to prepare-source.sh unchanged. dkms stops being a variant identity field. It was checked alongside srcpkg, binpkg and kernel_config, on the reasoning that those decide what the package is and only the destination varies. That holds for the others, which describe the kernel itself, but not for dkms: it names packages that have to exist in the target archive, so it is a property of the variant in a suite rather than of the variant. A variant's entries for one suite must still agree, which keeps the property worth having - a Daily that bundled a different module set from the Release it precedes would not be testing what ships - while letting the suites differ from each other. What each leg bundles is unchanged: the Debian suites get kgsl, camx and iris-vpu, resolute gets kgsl. It is now stated in the matrix rather than applied to it at build time. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-deb.yml | 21 +-------------------- README.md | 16 +++++++--------- ci/build-matrix.yaml | 13 ++++++++----- ci/scripts/resolve-matrix.py | 22 +++++++++++++++++++++- 4 files changed, 37 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml index 58e05bef..db87554a 100644 --- a/.github/workflows/build-kernel-deb.yml +++ b/.github/workflows/build-kernel-deb.yml @@ -194,8 +194,6 @@ jobs: prepare: name: Prepare kernel source runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] - # TEMPORARY, with the DKMS override in "Prepare source" below. - needs: resolve env: KERNEL_URL: ${{ inputs.git-clone || inputs.kernel-url || 'https://github.com/qualcomm-linux/kernel' }} @@ -378,29 +376,12 @@ jobs: KVER_EXTRA_INPUT: ${{ inputs.kver-extra }} KERNEL_CONFIG_INPUT: ${{ inputs.kernel-config }} DKMS_INPUT: ${{ inputs.dkms }} - # TEMPORARY, consumed by the DKMS override below. - SUITE_FAMILY: ${{ needs.resolve.outputs.family }} DEBUG_BUILD_INPUT: ${{ inputs.debug-build }} run: | # No -x here: keep xtrace off for this step as it assembles the # prepare-source.sh argument list which may include sensitive paths. set -euo pipefail - DKMS_LIST="$DKMS_INPUT" - - # TEMPORARY: camx and iris-vpu are bundled on non-Ubuntu builds only. - # A listed module is a presence contract, so an Ubuntu leg that asked - # for them would hard-fail rather than quietly ship without them. - # - # This is delivery policy and belongs in ci/build-matrix.yaml, whose - # entries are now one per suite and can carry it. Revert this - # override — and the "needs: resolve"/SUITE_FAMILY plumbing that - # feeds it — in the commit that moves the policy there. - if [[ "$SUITE_FAMILY" == "ubuntu" ]]; then - DKMS_LIST="kgsl" - echo "Ubuntu build: DKMS list overridden to '$DKMS_LIST' (was '$DKMS_INPUT')" - fi - ARGS=( --source-dir "$KERNEL_DIR" --distro "$DISTRO" @@ -414,7 +395,7 @@ jobs: [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") # Always passed, unlike the optional inputs above: an empty list is a # genuine "bundle nothing" rather than a request for some default. - ARGS+=(--dkms "$DKMS_LIST") + ARGS+=(--dkms "$DKMS_INPUT") [[ "$DEBUG_BUILD_INPUT" == "true" ]] && ARGS+=(--debug) docker run -i --rm \ diff --git a/README.md b/README.md index 6086a544..0ddb7327 100644 --- a/README.md +++ b/README.md @@ -182,17 +182,12 @@ them to the workflow matrix as they stand. Each entry carries: | `srcpkg` | Debian source package name. | | `binpkg` | Kernel image metapackage name. | | `kernel_config` | Extra fragments applied on top of `debian/config-available/`, all of which is applied to every build, one per list element. A bare name selects `debian/config-available/.config`; an `intree:` entry names a fragment shipped by the kernel source, as a path relative to the kernel source root (e.g. `intree:arch/arm64/configs/qcom_debug.config`), so it stays versioned with the kernel it targets. Empty for variants that need nothing beyond `config-available/`; today it carries only `intree:` fragments. `resolve-matrix.py` joins it into the comma-separated `kernel-config` workflow input. | -| `dkms` | Out-of-tree DKMS modules built and bundled into the image package, one per list element, each named without the `-dkms` suffix (e.g. `kgsl`). An empty list bundles nothing. A listed module is a presence contract: a build fails rather than shipping an image without it. `resolve-matrix.py` joins it into the comma-separated `dkms` workflow input, which reaches `prepare-source.sh --dkms`; see [debian/README.md](debian/README.md) for what the packaging does with it. | +| `dkms` | Out-of-tree DKMS modules built and bundled into the image package, one per list element, each named without the `-dkms` suffix (e.g. `kgsl`). Each needs a `-dkms` package in the suite being built for, so this varies between suites. An empty list bundles nothing. A listed module is a presence contract: a build fails rather than shipping an image without it. `resolve-matrix.py` joins it into the comma-separated `dkms` workflow input, which reaches `prepare-source.sh --dkms`; see [debian/README.md](debian/README.md) for what the packaging does with it. | | `debian_revision` | The Debian revision this package is built at, stated outright. Daily revisions end in `~`; Release revisions do not. | | `localversion`, `kver_extra` | Optional version overrides forwarded to packaging. | | `debusine_parent_workspace` | Optional parent workspace override for the variant's CI child workspaces. | | `target_workspace` | Debusine destination for Release entries only. | -`dkms` currently has one exception the matrix cannot express: `build-kernel-deb.yml` -replaces the resolved list with `kgsl` on Ubuntu-family legs, so `camx` and -`iris-vpu` are bundled on Debian suites only. That override is temporary and goes -away once the per-suite `dkms` lists the flattened matrix now allows are used. - `resolve-matrix.py` rejects the matrix — before any build job starts — where an entry has an unknown field or a missing required one, a malformed variant or suite identifier, a `ref_strategy` its `type` does not allow (`Daily` must @@ -209,9 +204,12 @@ face value: - No two entries share a `kernel_variant`, `type` and `suite` — one entry is one generated package. -- A variant's entries agree on `srcpkg`, `binpkg`, `kernel_config` and `dkms`; - those decide what the package *is*, and the entries differ only in where it - goes. +- A variant's entries agree on `srcpkg`, `binpkg` and `kernel_config`; those + decide what the package *is*, and the entries differ only in where it goes. +- A variant's entries for one suite agree on `dkms`. The module set depends on + which `-dkms` packages the target archive carries, so it varies between + suites — but a suite's Daily and Release must match, or the Daily is not + testing the module set the Release will ship. - A variant's entries of one `type` agree on `git_clone`, `branch_or_tag`, `ref_strategy` and `tag_pattern`, so a forgotten suite cannot quietly ship a different kernel from its siblings after a release ref bump. diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index 8bb0e556..2ce3feae 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -13,8 +13,13 @@ # Entries are written out in full rather than sharing YAML anchors, so each one # can be read, grepped, and changed on its own. resolve-matrix.py enforces the # consistency that duplication would otherwise put at risk: a variant's entries -# must agree on srcpkg, binpkg and kernel_config, and its entries of one -# delivery type must agree on the kernel ref they build. +# must agree on srcpkg, binpkg and kernel_config, its entries of one delivery +# type must agree on the kernel ref they build, and its entries for one suite +# must agree on dkms. +# +# dkms varies by suite because it depends on which -dkms packages the +# target archive carries: the Debian suites bundle camx and iris-vpu, which +# Ubuntu does not package. # # debian_revision is stated outright, not derived. Daily revisions end in "~" # so they sort below the Release revision for the same suite; the rest of the @@ -70,11 +75,9 @@ deliveries: srcpkg: linux-qcom-next binpkg: linux-image-qcom-next kernel_config: [] + # kgsl alone: camx-dkms and iris-vpu-dkms are not packaged for Ubuntu. dkms: - kgsl - - camx - - iris-vpu - - audioreach debian_revision: '0qli1~26.04.1~' - kernel_variant: qcom-next diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index 4f97703c..f92342cf 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -100,7 +100,14 @@ # Fields that identify the variant itself rather than one of its build legs. # Every entry for a variant must agree on them, because they decide what the # package is; the entries only differ in where it is delivered. -VARIANT_IDENTITY_FIELDS = ("srcpkg", "binpkg", "kernel_config", "dkms") +VARIANT_IDENTITY_FIELDS = ("srcpkg", "binpkg", "kernel_config") + +# Fields a variant may vary between suites but not within one. The out-of-tree +# module set depends on which -dkms packages the target archive has, so +# it is a property of the variant in a suite rather than of the variant. A +# suite's Daily and Release must still agree: a Daily that bundles a different +# module set from the Release it precedes is not testing what will ship. +SUITE_IDENTITY_FIELDS = ("dkms",) # Fields deciding which kernel tree is built. All entries for one variant and # delivery type build the same source, so a stale suite cannot quietly ship a @@ -289,6 +296,7 @@ def check_consistency(deliveries, errors): by_variant_type = defaultdict(list) by_package_version = defaultdict(list) variants_by_package = defaultdict(set) + by_variant_suite = defaultdict(list) for entry in deliveries: if not isinstance(entry, dict): @@ -301,6 +309,7 @@ def check_consistency(deliveries, errors): by_variant[variant].append(entry) by_variant_type[(variant, delivery_type)].append(entry) + by_variant_suite[(variant, suite)].append(entry) by_leg[(variant, delivery_type, suite)].append(entry) for field in ("srcpkg", "binpkg"): @@ -337,6 +346,17 @@ def check_consistency(deliveries, errors): "every variant must define at least one of each" ) + for (variant, suite), entries in sorted( + by_variant_suite.items(), key=lambda item: str(item[0]) + ): + for field in SUITE_IDENTITY_FIELDS: + values = {json.dumps(entry.get(field), sort_keys=True) for entry in entries} + if len(values) > 1: + errors.append( + f"kernel_variant {variant} must use one {field} across its " + f"{suite} entries (got " + ", ".join(sorted(values)) + ")" + ) + for (variant, delivery_type), entries in sorted( by_variant_type.items(), key=lambda item: str(item[0]) ): From b526b3cb6be5a47addcb7267af39403e2985b125 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 15/71] ci: stop the build job name nesting variant above suite The matrix build job was named " / ". GitHub composes a called workflow's job names as " / " and renders the result as a tree, so the separator inside the caller's own name added a level that means nothing: every run grew a qcom-next hub and a qcom-next-debug hub, each holding its suites, when the thing being run is one leg per suite. Name the job " ()" instead, which is how GitHub writes its own matrix job names, so a leg reads as one entry. The jobs themselves are unchanged. The remaining nesting under each leg is the reusable workflow's own jobs, which GitHub always surfaces individually. Signed-off-by: Christopher Obbard --- .github/workflows/daily.yml | 2 +- .github/workflows/pr-build.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 1e30522c..f6b85be0 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -109,7 +109,7 @@ jobs: # build does not cancel the forky build. # --------------------------------------------------------------------------- build: - name: ${{ matrix.kernel_variant }} / ${{ matrix.suite }} + name: ${{ matrix.kernel_variant }} (${{ matrix.suite }}) needs: configure-matrix strategy: fail-fast: false diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 2d2888ae..dd964214 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -52,7 +52,7 @@ jobs: # resolute results on the PR. # --------------------------------------------------------------------------- build: - name: ${{ matrix.kernel_variant }} / ${{ matrix.suite }} + name: ${{ matrix.kernel_variant }} (${{ matrix.suite }}) needs: configure-matrix strategy: fail-fast: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 62e31eea..cc7693df 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,7 +93,7 @@ jobs: # release path (lib/release) instead of the S3 publish path. # --------------------------------------------------------------------------- build: - name: ${{ matrix.kernel_variant }} / ${{ matrix.suite }} + name: ${{ matrix.kernel_variant }} (${{ matrix.suite }}) needs: configure-matrix strategy: fail-fast: false From c65cee6f6be69f0b91359e1804e20053fcf675d7 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 16/71] ci: keep kernel_variant out of the kernel version kernel_variant was doing two unrelated jobs. It named the build in CI - the Actions job, the prepared-source artifact, the Debusine child workspace, the S3 path - and it was also handed to derive-localversion.sh, which folded it into LOCALVERSION and so into the kernel release string and the versioned linux-image package name. Renaming a build leg therefore renamed the kernel that leg installs, which is not something a CI label should be able to do. Split the two. Add a flavour field: the kernel's own identity, the part of the release string that distinguishes two kernels built from one ref with different configuration so their linux-image packages coexist. derive-localversion.sh takes --flavour in place of --variant, and build-kernel-deb.yml feeds it from the new flavour input rather than kernel-variant. kernel_variant keeps every CI-facing use and reaches nothing that ends up in a package. Everything the matrix asserts about what is built is regrouped onto flavour, since that is now what identifies a package: the srcpkg, binpkg and kernel_config agreement, the per-suite dkms agreement, the kernel ref agreement within a delivery type, the Daily-and-Release requirement, the srcpkg and binpkg exclusivity, and the Daily/Release revision pairing. kernel_variant keeps one check of its own, that it does not repeat for a type and suite, which is what makes job names and artifact scopes unique. The flavours are qcom-next and qcom-next-debug, the values kernel_variant already held, so every leg builds the same kernel release it did before: verified leg by leg against the old script, LOCALVERSION identical for all nine. kernel_variant is now free. Renaming the legs to qcom-next-trixie and so on validates cleanly and leaves LOCALVERSION untouched, which it could not have done before this change. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-deb.yml | 17 +++-- .github/workflows/daily.yml | 6 +- .github/workflows/pr-build.yml | 1 + .github/workflows/release.yml | 1 + README.md | 38 ++++++----- ci/build-matrix.yaml | 19 +++++- ci/scripts/derive-localversion.sh | 36 ++++++----- ci/scripts/resolve-matrix.py | 88 +++++++++++++++----------- 8 files changed, 133 insertions(+), 73 deletions(-) diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml index db87554a..f620d365 100644 --- a/.github/workflows/build-kernel-deb.yml +++ b/.github/workflows/build-kernel-deb.yml @@ -4,7 +4,12 @@ on: workflow_dispatch: inputs: kernel-variant: - description: 'Kernel variant identifier' + description: 'Kernel variant identifier, naming this build in CI only' + required: true + default: 'qcom-next' + type: string + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' required: true default: 'qcom-next' type: string @@ -82,7 +87,11 @@ on: workflow_call: inputs: kernel-variant: - description: 'Kernel variant identifier' + description: 'Kernel variant identifier, naming this build in CI only' + type: string + default: 'qcom-next' + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' type: string default: 'qcom-next' suite: @@ -278,7 +287,7 @@ jobs: - name: Derive LOCALVERSION env: LOCALVERSION_INPUT: ${{ inputs.localversion }} - KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} + FLAVOUR_INPUT: ${{ inputs.flavour || 'qcom-next' }} run: | set -euo pipefail OVERRIDE_LV="$LOCALVERSION_INPUT" @@ -287,7 +296,7 @@ jobs: echo "LOCALVERSION=$OVERRIDE_LV" >> "$GITHUB_ENV" else LOCALVERSION=$(ci/scripts/derive-localversion.sh \ - --variant "$KERNEL_VARIANT_INPUT" \ + --flavour "$FLAVOUR_INPUT" \ --ref "$KERNEL_REF" \ --sha "$(echo "$KERNEL_SHA" | cut -c1-12)") echo "LOCALVERSION=$LOCALVERSION" >> "$GITHUB_ENV" diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index f6b85be0..e5d7cf4b 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -9,9 +9,10 @@ name: daily # Matrix schema: ci/build-matrix.yaml # One entry is one generated package, so ci/scripts/resolve-matrix.py selects # the entries with type: Daily and passes them through as they stand. Every -# field (kernel_variant, suite, srcpkg, binpkg, kernel_config, dkms, +# field (kernel_variant, suite, flavour, srcpkg, binpkg, kernel_config, dkms, # debian_revision, git_clone, branch_or_tag, ref_strategy) becomes a -# build-kernel-deb.yml input. +# build-kernel-deb.yml input. kernel_variant names the job and scopes its +# artifacts; flavour is what reaches the built kernel's version. # # To add a daily build target: add an entry to ci/build-matrix.yaml with # type: Daily. No workflow changes needed. @@ -119,6 +120,7 @@ jobs: with: suite: ${{ matrix.suite }} kernel-variant: ${{ matrix.kernel_variant }} + flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} ref-strategy: ${{ matrix.ref_strategy }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index dd964214..4ef4263a 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -62,6 +62,7 @@ jobs: with: suite: ${{ matrix.suite }} kernel-variant: ${{ matrix.kernel_variant }} + flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} ref-strategy: ${{ matrix.ref_strategy }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cc7693df..80707bc6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -103,6 +103,7 @@ jobs: with: suite: ${{ matrix.suite }} kernel-variant: ${{ matrix.kernel_variant }} + flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} ref-strategy: ${{ matrix.ref_strategy }} diff --git a/README.md b/README.md index 0ddb7327..4c7b452a 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,8 @@ kernel variants, each with independent source/package identity, kernel source and ref strategy, configuration fragments, Debian revision, target suites, and release destination. -Every kernel variant owns exactly two complete matrix rows: one `Daily` row and -one `Release` row. The resolver expands every suite in those rows into an -isolated `kernel_variant + suite` build leg. +Every entry in the matrix is one isolated build leg producing one package. A +kernel flavour owns as many entries as it has suites and delivery types. ### Configured variants @@ -26,11 +25,13 @@ isolated `kernel_variant + suite` build leg. | `qcom-next` | `linux-qcom-next` | `linux-image-qcom-next` | trixie, forky, resolute | trixie, forky | Standard kernel | | `qcom-next-debug` | `linux-qcom-next-debug` | `linux-image-qcom-next-debug` | trixie, forky | trixie, forky | Adds `arch/arm64/configs/qcom_debug.config` and `kernel/configs/debug.config` from the kernel source, via `intree:` entries | -Both build the same kernel ref. `derive-localversion.sh` folds the variant name +Both build the same kernel ref. `derive-localversion.sh` folds the *flavour* into LOCALVERSION, so each produces a distinct kernel release (`+qcom-next--g` and `+qcom-next-debug--g`) and therefore a distinct versioned image package that can be installed alongside the other. See [docs/version.md](docs/version.md) for how the version strings are composed. +The flavour is what the kernel is; `kernel_variant` is only what CI calls the +build. `ci/build-matrix.yaml` is the source of truth; this table is a summary. @@ -50,6 +51,7 @@ deliveries: - kernel_variant: qcom-next type: Daily suite: trixie + flavour: qcom-next git_clone: https://github.com/qualcomm-linux/kernel branch_or_tag: qcom-next ref_strategy: latest_tag @@ -67,6 +69,7 @@ deliveries: - kernel_variant: qcom-next type: Release suite: trixie + flavour: qcom-next git_clone: https://github.com/qualcomm-linux/kernel branch_or_tag: ref_strategy: pinned_ref @@ -172,9 +175,10 @@ them to the workflow matrix as they stand. Each entry carries: | Field | Purpose | | --- | --- | -| `kernel_variant` | Stable identifier for a separately packaged kernel variant. Lowercase letters, digits, and internal hyphens only. | +| `kernel_variant` | CI identifier for this build: its Actions job name, prepared-source artifact, Debusine child workspace and S3 path. Never reaches a package name or version. Lowercase letters, digits, and internal hyphens only. | | `type` | `Daily` or `Release`. | | `suite` | The one suite this entry builds for. | +| `flavour` | The kernel's own identity, and the only matrix field that reaches the built kernel. `derive-localversion.sh` makes it the LOCALVERSION suffix, so two flavours built from one ref get distinct kernel releases and their `linux-image` packages coexist. All entries sharing a flavour build the same package for different suites. | | `git_clone` | Kernel source repository. | | `branch_or_tag` | Source branch or pinned tag, according to `ref_strategy`. | | `ref_strategy` | `latest_tag`, `branch_tip`, or `pinned_ref`. | @@ -204,17 +208,17 @@ face value: - No two entries share a `kernel_variant`, `type` and `suite` — one entry is one generated package. -- A variant's entries agree on `srcpkg`, `binpkg` and `kernel_config`; those +- A flavour's entries agree on `srcpkg`, `binpkg` and `kernel_config`; those decide what the package *is*, and the entries differ only in where it goes. -- A variant's entries for one suite agree on `dkms`. The module set depends on +- A flavour's entries for one suite agree on `dkms`. The module set depends on which `-dkms` packages the target archive carries, so it varies between suites — but a suite's Daily and Release must match, or the Daily is not testing the module set the Release will ship. -- A variant's entries of one `type` agree on `git_clone`, `branch_or_tag`, +- A flavour's entries of one `type` agree on `git_clone`, `branch_or_tag`, `ref_strategy` and `tag_pattern`, so a forgotten suite cannot quietly ship a different kernel from its siblings after a release ref bump. -- Every variant defines at least one `Daily` and one `Release` entry. -- No `srcpkg` or `binpkg` is shared between variants, and no two entries build +- Every flavour defines at least one `Daily` and one `Release` entry. +- No `srcpkg` or `binpkg` is shared between flavours, and no two entries build the same `srcpkg` at the same `debian_revision`. - Where a suite has both, its `Daily` revision is its `Release` revision plus a trailing `~`. @@ -246,7 +250,7 @@ Supporting scripts keep workflow YAML small and testable: | --- | --- | | `ci/scripts/resolve-matrix.py` | Validates the delivery matrix and selects the entries to build. Needs PyYAML (`python3-yaml`). | | `ci/scripts/resolve-kernel-ref.sh` | Resolves a matrix-selected dated tag or validates a direct ref. | -| `ci/scripts/derive-localversion.sh` | Derives the version fields from the variant, resolved kernel ref and HEAD, printing `LOCALVERSION=`, `SNAPSHOT=` and `GITSHA=` lines. `SNAPSHOT` is the dated component of the Debian version: the tag's date, or the HEAD commit date for a branch-tip build. Scheme and rationale: [docs/version.md](docs/version.md). | +| `ci/scripts/derive-localversion.sh` | Derives the version fields from the flavour, resolved kernel ref and HEAD, printing `LOCALVERSION=`, `SNAPSHOT=` and `GITSHA=` lines. `SNAPSHOT` is the dated component of the Debian version: the tag's date, or the HEAD commit date for a branch-tip build. Scheme and rationale: [docs/version.md](docs/version.md). | ## Architecture @@ -448,7 +452,8 @@ The available inputs are: | Input | Default | Purpose | | --- | --- | --- | -| `kernel-variant` | `qcom-next` | Stable variant identifier used in artifact and workspace identity. | +| `kernel-variant` | `qcom-next` | CI identifier used for artifact and workspace identity only. | +| `flavour` | `qcom-next` | Kernel flavour: the LOCALVERSION suffix, and so the kernel release identity. | | `suite` | `trixie` | Target suite. | | `ref-strategy` | `latest_tag` | `latest_tag`, `branch_tip`, or `pinned_ref`. | | `kernel-branch` | `qcom-next` | Branch for `branch_tip`, or immutable ref for `pinned_ref`; ignored by `latest_tag`. | @@ -497,9 +502,12 @@ To add a kernel variant: 1. Add one entry to `deliveries` per package the variant should produce: one per Daily suite and one per Release suite, each spelling out all of its own fields. Do not rely on another variant's values. -2. Keep `srcpkg`, `binpkg` and `kernel_config` identical across every entry for - the variant, and the ref fields identical across its entries of one `type`. - `resolve-matrix.py` rejects the matrix if they drift apart. +2. Give them all the same `flavour`, distinct from every other flavour's — it + becomes the LOCALVERSION suffix, so this is what lets the new kernel install + alongside the existing ones. Keep `srcpkg`, `binpkg` and `kernel_config` + identical across every entry for the flavour, and the ref fields identical + across its entries of one `type`. `resolve-matrix.py` rejects the matrix if + they drift apart. 3. Use `latest_tag` with a dated tag glob or `branch_tip` for Daily. Use `pinned_ref` for Release, and update that ref through a reviewed PR. 4. Give the variant distinct `srcpkg` and `binpkg` values. Set diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index 2ce3feae..d4693b50 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -12,11 +12,19 @@ # # Entries are written out in full rather than sharing YAML anchors, so each one # can be read, grepped, and changed on its own. resolve-matrix.py enforces the -# consistency that duplication would otherwise put at risk: a variant's entries +# consistency that duplication would otherwise put at risk: a flavour's entries # must agree on srcpkg, binpkg and kernel_config, its entries of one delivery # type must agree on the kernel ref they build, and its entries for one suite # must agree on dkms. # +# The two identifiers are not the same thing. flavour is the kernel's own +# identity: it becomes the LOCALVERSION suffix, so qcom-next and +# qcom-next-debug produce distinct kernel releases that install alongside each +# other, and every suite building one flavour produces the same kernel release. +# kernel_variant names the build in CI - the Actions job, the prepared-source +# artifact, the Debusine workspace, the S3 path - and never reaches a package +# name or version. +# # dkms varies by suite because it depends on which -dkms packages the # target archive carries: the Debian suites bundle camx and iris-vpu, which # Ubuntu does not package. @@ -34,6 +42,7 @@ deliveries: - kernel_variant: qcom-next type: Daily suite: trixie + flavour: qcom-next git_clone: https://github.com/qualcomm-linux/kernel branch_or_tag: qcom-next ref_strategy: latest_tag @@ -51,6 +60,7 @@ deliveries: - kernel_variant: qcom-next type: Daily suite: forky + flavour: qcom-next git_clone: https://github.com/qualcomm-linux/kernel branch_or_tag: qcom-next ref_strategy: latest_tag @@ -68,6 +78,7 @@ deliveries: - kernel_variant: qcom-next type: Daily suite: resolute + flavour: qcom-next git_clone: https://github.com/qualcomm-linux/kernel branch_or_tag: qcom-next ref_strategy: latest_tag @@ -83,6 +94,7 @@ deliveries: - kernel_variant: qcom-next type: Release suite: trixie + flavour: qcom-next git_clone: https://github.com/qualcomm-linux/kernel branch_or_tag: qcom-next-7.2-rc7-20260821 ref_strategy: pinned_ref @@ -100,6 +112,7 @@ deliveries: - kernel_variant: qcom-next type: Release suite: forky + flavour: qcom-next git_clone: https://github.com/qualcomm-linux/kernel branch_or_tag: qcom-next-7.2-rc7-20260821 ref_strategy: pinned_ref @@ -121,6 +134,7 @@ deliveries: - kernel_variant: qcom-next-debug type: Daily suite: trixie + flavour: qcom-next-debug git_clone: https://github.com/qualcomm-linux/kernel branch_or_tag: qcom-next ref_strategy: latest_tag @@ -140,6 +154,7 @@ deliveries: - kernel_variant: qcom-next-debug type: Daily suite: forky + flavour: qcom-next-debug git_clone: https://github.com/qualcomm-linux/kernel branch_or_tag: qcom-next ref_strategy: latest_tag @@ -159,6 +174,7 @@ deliveries: - kernel_variant: qcom-next-debug type: Release suite: trixie + flavour: qcom-next-debug git_clone: https://github.com/qualcomm-linux/kernel branch_or_tag: qcom-next-7.2-rc7-20260821 ref_strategy: pinned_ref @@ -178,6 +194,7 @@ deliveries: - kernel_variant: qcom-next-debug type: Release suite: forky + flavour: qcom-next-debug git_clone: https://github.com/qualcomm-linux/kernel branch_or_tag: qcom-next-7.2-rc7-20260821 ref_strategy: pinned_ref diff --git a/ci/scripts/derive-localversion.sh b/ci/scripts/derive-localversion.sh index eca3c989..0517e10e 100755 --- a/ci/scripts/derive-localversion.sh +++ b/ci/scripts/derive-localversion.sh @@ -3,23 +3,29 @@ # SPDX-License-Identifier: BSD-3-Clause-Clear set -euo pipefail -# Derive the version fields for a build from a kernel variant and resolved ref. +# Derive the version fields for a build from a kernel flavour and resolved ref. +# +# The flavour is the kernel's own identity, the part of the kernel release +# that distinguishes two kernels built from the same ref with different +# configuration, so that their linux-image packages install alongside each +# other. It is not the CI identifier for the build: a flavour is built for +# several suites, and all of those builds produce the same kernel release. # # Emits LOCALVERSION (the kernel release suffix), SNAPSHOT (the dated component # of the Debian version) and GITSHA, all derived from the ref in one place. # SNAPSHOT and GITSHA are emitted alongside rather than recovered from # LOCALVERSION later: reading them back out means guessing where each field ends -# in a string that also carries a variant name, and a hex SHA can end in eight +# in a string that also carries a flavour name, and a hex SHA can end in eight # digits of its own. # # For dated tag builds (ref ends in -YYYYMMDD, optionally .): -# Produces +-[.]-g<12 hex>. +# Produces +-[.]-g<12 hex>. # Example: qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722-g07f50dc44edd # qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1-g07f50dc44edd # # The respin ordinal distinguishes a second tag cut on the same day. It is # carried verbatim rather than normalised, so the first tag of a day stays -# plain +-: systemd compares the separator before the chunk +# plain +-: systemd compares the separator before the chunk # behind it, so an absent ordinal already sorts below a present one and no # build has to spell a ".0". # @@ -61,11 +67,11 @@ set -euo pipefail # which spells the release candidate ~rcN and orders correctly either way. # # Usage: -# ci/scripts/derive-localversion.sh --variant qcom-next --ref qcom-next-7.2-rc3-20260722 --sha 07f50dc44edd -# ci/scripts/derive-localversion.sh --variant arduino --ref main --sha 07f50dc44edd --date 20260904 +# ci/scripts/derive-localversion.sh --flavour qcom-next --ref qcom-next-7.2-rc3-20260722 --sha 07f50dc44edd +# ci/scripts/derive-localversion.sh --flavour arduino --ref main --sha 07f50dc44edd --date 20260904 # # Options: -# --variant VARIANT Kernel variant identifier. Defaults to qcom-next. +# --flavour FLAVOUR Kernel flavour. Defaults to qcom-next. # --ref REF Kernel ref (tag name or branch name). Required. # --sha SHA Commit SHA, truncated to 12 hex characters. Required. # --date DATE HEAD commit date as YYYYMMDD or YYYYMMDD.N. Required for @@ -86,7 +92,7 @@ set -euo pipefail # 0 Success. # 1 Error (invalid args, malformed --sha, branch tip without --date). -VARIANT="qcom-next" +FLAVOUR="qcom-next" REF="" SHA="" DATE="" @@ -98,7 +104,7 @@ usage() { while [[ $# -gt 0 ]]; do case $1 in - --variant) VARIANT="$2"; shift 2 ;; + --flavour) FLAVOUR="$2"; shift 2 ;; --ref) REF="$2"; shift 2 ;; --sha) SHA="$2"; shift 2 ;; --date) DATE="$2"; shift 2 ;; @@ -108,8 +114,8 @@ while [[ $# -gt 0 ]]; do done [[ -n "$REF" ]] || { echo "ERROR: --ref is required" >&2; exit 1; } -[[ "$VARIANT" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { - echo "ERROR: --variant must use lowercase letters, digits, and internal hyphens" >&2 +[[ "$FLAVOUR" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { + echo "ERROR: --flavour must use lowercase letters, digits, and internal hyphens" >&2 exit 1 } # Every build identifies its commit, so --sha is required for all of them, not @@ -123,11 +129,11 @@ done GITSHA="${SHA:0:12}" # Dated tags use a trailing YYYYMMDD snapshot, optionally followed by a respin -# ordinal. The matrix selects the tag set; the variant supplies the stable -# package identity used in LOCALVERSION. +# ordinal. The matrix selects the tag set; the flavour supplies the stable +# kernel identity used in LOCALVERSION. if [[ "$REF" =~ -([0-9]{8}(\.[0-9]+)?)$ ]]; then SNAPSHOT="${BASH_REMATCH[1]}" - LOCALVERSION="+${VARIANT}-${SNAPSHOT}-g${GITSHA}" + LOCALVERSION="+${FLAVOUR}-${SNAPSHOT}-g${GITSHA}" else # Branch-tip build: the ref carries no date, so the commit date supplies # one. Without it these builds had no snapshot at all, which put their @@ -141,7 +147,7 @@ else exit 1 } SNAPSHOT="$DATE" - LOCALVERSION="+${VARIANT}-${SNAPSHOT}-g${GITSHA}" + LOCALVERSION="+${FLAVOUR}-${SNAPSHOT}-g${GITSHA}" fi echo "LOCALVERSION=${LOCALVERSION}" diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index f92342cf..9c7eed52 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -77,6 +77,7 @@ "kernel_variant", "type", "suite", + "flavour", "git_clone", "branch_or_tag", "ref_strategy", @@ -97,19 +98,26 @@ REQUIRED_STRING_FIELDS + OPTIONAL_STRING_FIELDS + ("kernel_config", "dkms") ) -# Fields that identify the variant itself rather than one of its build legs. -# Every entry for a variant must agree on them, because they decide what the +# Everything about what is built is anchored on flavour, not kernel_variant. +# The flavour is the kernel's own identity: it becomes the LOCALVERSION +# suffix, so two flavours built from one ref produce distinct kernel releases +# whose linux-image packages install alongside each other. kernel_variant +# names the build in CI -- job names, artifacts, Debusine workspaces, S3 +# paths -- and carries no packaging meaning. + +# Fields that identify the flavour itself rather than one of its build legs. +# Every entry for a flavour must agree on them, because they decide what the # package is; the entries only differ in where it is delivered. -VARIANT_IDENTITY_FIELDS = ("srcpkg", "binpkg", "kernel_config") +FLAVOUR_IDENTITY_FIELDS = ("srcpkg", "binpkg", "kernel_config") -# Fields a variant may vary between suites but not within one. The out-of-tree +# Fields a flavour may vary between suites but not within one. The out-of-tree # module set depends on which -dkms packages the target archive has, so -# it is a property of the variant in a suite rather than of the variant. A +# it is a property of the flavour in a suite rather than of the flavour. A # suite's Daily and Release must still agree: a Daily that bundles a different # module set from the Release it precedes is not testing what will ship. SUITE_IDENTITY_FIELDS = ("dkms",) -# Fields deciding which kernel tree is built. All entries for one variant and +# Fields deciding which kernel tree is built. All entries for one flavour and # delivery type build the same source, so a stale suite cannot quietly ship a # different kernel from its siblings. REF_FIELDS = ("git_clone", "branch_or_tag", "ref_strategy", "tag_pattern") @@ -222,7 +230,7 @@ def check_entry(entry, report): for field in sorted(set(entry) - KNOWN_FIELDS): report(f"unknown field {field}") - for field in ("kernel_variant", "suite"): + for field in ("kernel_variant", "suite", "flavour"): value = entry.get(field) if isinstance(value, str) and not NAME_RE.match(value): report(f"{field} must use lowercase letters, digits, and internal hyphens") @@ -287,40 +295,47 @@ def describe(entry, index): def check_consistency(deliveries, errors): """Validate the invariants that span entries. - Entries are written out in full, so the matrix can state a variant twice + Entries are written out in full, so the matrix can state a flavour twice and disagree with itself. These checks are what makes that duplication safe to read at face value. + + Everything about the package is grouped by flavour. kernel_variant is + only checked for the one thing it is used for: naming a build leg + uniquely. """ by_leg = defaultdict(list) - by_variant = defaultdict(list) - by_variant_type = defaultdict(list) + by_flavour = defaultdict(list) + by_flavour_type = defaultdict(list) + by_flavour_suite = defaultdict(list) by_package_version = defaultdict(list) - variants_by_package = defaultdict(set) - by_variant_suite = defaultdict(list) + flavours_by_package = defaultdict(set) for entry in deliveries: if not isinstance(entry, dict): continue - variant = entry.get("kernel_variant") + flavour = entry.get("flavour") delivery_type = entry.get("type") suite = entry.get("suite") - if not isinstance(variant, str): + if not isinstance(flavour, str): continue - by_variant[variant].append(entry) - by_variant_type[(variant, delivery_type)].append(entry) - by_variant_suite[(variant, suite)].append(entry) - by_leg[(variant, delivery_type, suite)].append(entry) + by_flavour[flavour].append(entry) + by_flavour_type[(flavour, delivery_type)].append(entry) + by_flavour_suite[(flavour, suite)].append(entry) + by_leg[(entry.get("kernel_variant"), delivery_type, suite)].append(entry) for field in ("srcpkg", "binpkg"): if isinstance(entry.get(field), str) and entry[field]: - variants_by_package[(field, entry[field])].add(variant) + flavours_by_package[(field, entry[field])].add(flavour) if isinstance(entry.get("srcpkg"), str) and isinstance( entry.get("debian_revision"), str ): by_package_version[(entry["srcpkg"], entry["debian_revision"])].append(entry) + # kernel_variant plus suite is what names a build leg in the Actions UI and + # scopes its artifacts, workspace and S3 path, so a repeat would have two + # legs writing to one another's outputs. for (variant, delivery_type, suite), entries in sorted( by_leg.items(), key=lambda item: str(item[0]) ): @@ -330,50 +345,51 @@ def check_consistency(deliveries, errors): f"entries for suite {suite}; one entry is one generated package" ) - for variant, entries in sorted(by_variant.items()): - for field in VARIANT_IDENTITY_FIELDS: + for flavour, entries in sorted(by_flavour.items()): + for field in FLAVOUR_IDENTITY_FIELDS: values = {json.dumps(entry.get(field), sort_keys=True) for entry in entries} if len(values) > 1: errors.append( - f"kernel_variant {variant} must use one {field} across all its " + f"flavour {flavour} must use one {field} across all its " "entries (got " + ", ".join(sorted(values)) + ")" ) types = {entry.get("type") for entry in entries} for delivery_type in DELIVERY_TYPES: if delivery_type not in types: errors.append( - f"kernel_variant {variant} has no {delivery_type} entry; " - "every variant must define at least one of each" + f"flavour {flavour} has no {delivery_type} entry; " + "every flavour must define at least one of each" ) - for (variant, suite), entries in sorted( - by_variant_suite.items(), key=lambda item: str(item[0]) + for (flavour, suite), entries in sorted( + by_flavour_suite.items(), key=lambda item: str(item[0]) ): for field in SUITE_IDENTITY_FIELDS: values = {json.dumps(entry.get(field), sort_keys=True) for entry in entries} if len(values) > 1: errors.append( - f"kernel_variant {variant} must use one {field} across its " + f"flavour {flavour} must use one {field} across its " f"{suite} entries (got " + ", ".join(sorted(values)) + ")" ) - for (variant, delivery_type), entries in sorted( - by_variant_type.items(), key=lambda item: str(item[0]) + for (flavour, delivery_type), entries in sorted( + by_flavour_type.items(), key=lambda item: str(item[0]) ): for field in REF_FIELDS: values = {entry.get(field) for entry in entries} if len(values) > 1: rendered = ", ".join(sorted(str(v) for v in values)) errors.append( - f"kernel_variant {variant} must build one {field} across its " + f"flavour {flavour} must build one {field} across its " f"{delivery_type} entries (got {rendered})" ) - for (field, package), variants in sorted(variants_by_package.items()): - if len(variants) > 1: + # Two flavours sharing a package name would overwrite each other in the + # archive; the whole point of a second flavour is a second package. + for (field, package), flavours in sorted(flavours_by_package.items()): + if len(flavours) > 1: errors.append( - f"{field} {package} is shared by kernel variants " - + ", ".join(sorted(variants)) + f"{field} {package} is shared by flavours " + ", ".join(sorted(flavours)) ) for (srcpkg, revision), entries in sorted( @@ -388,7 +404,7 @@ def check_consistency(deliveries, errors): # A suite's Daily and Release differ only by the Daily's trailing ~, so the # Daily reliably sorts below the Release that supersedes it. - for variant, entries in sorted(by_variant.items()): + for flavour, entries in sorted(by_flavour.items()): revisions = { (entry.get("type"), entry.get("suite")): entry.get("debian_revision") for entry in entries @@ -401,7 +417,7 @@ def check_consistency(deliveries, errors): release = revisions.get(("Release", suite)) if isinstance(release, str) and revision != release + "~": errors.append( - f"kernel_variant {variant} suite {suite}: Daily " + f"flavour {flavour} suite {suite}: Daily " f'debian_revision "{revision}" must be the Release revision ' f'"{release}" with a trailing ~' ) From da019162c8a20569c214a1e30b06df51accf84f7 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 17/71] ci: name each build leg with its own kernel_variant kernel_variant repeated: three Daily entries all called qcom-next, distinguished only by their suite. The Actions job had to be named from two fields to say which leg it was, and a dispatch had to take a variant and a suite and offer scopes for each combination of the two. Give every entry a kernel_variant of its own - qcom-next-trixie, qcom-next-debug-forky - so a variant names exactly one build. The job name is now just the variant, a dispatch asks for builds by name, and resolve-matrix.py rejects a matrix that uses one variant twice in a delivery type. Uniqueness is per type rather than global: a run only ever resolves one type, so a Daily and the Release that supersedes it keep the same name, which is what makes them recognisable as the same leg. The dispatch filters take comma-separated lists, so a run can name several legs. --flavour joins them, because "release qcom-next everywhere it ships" is the normal release action and naming each suite individually would be a worse way to ask for it. A name matching no entry is now an error listing what is available, rather than a silently smaller build set - with per-leg names there is much more to mistype. Daily scopes become Full matrix, Selected variants, and Selected flavour; Release becomes Selected flavour and Selected variants. Renaming the legs would have moved every published artifact, since the S3 path, Debusine workspace and prepared-source artifact were all keyed on kernel_variant. They describe which package is at that location, not which CI job wrote it, so they are keyed on flavour now and their values are unchanged: qcom-next/trixie stays qcom-next/trixie. Verified leg by leg, along with LOCALVERSION. build-kernel-debusine.yml and build-kernel-ubuntu.yml take flavour in place of kernel-variant and no longer see the CI label at all. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-deb.yml | 8 +- .github/workflows/build-kernel-debusine.yml | 10 +- .github/workflows/build-kernel-ubuntu.yml | 8 +- .github/workflows/daily.yml | 46 ++++---- .github/workflows/pr-build.yml | 2 +- .github/workflows/release.yml | 42 +++---- README.md | 51 +++++---- ci/build-matrix.yaml | 18 +-- ci/scripts/resolve-matrix.py | 119 +++++++++++++++----- 9 files changed, 189 insertions(+), 115 deletions(-) diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml index f620d365..93c16fa3 100644 --- a/.github/workflows/build-kernel-deb.yml +++ b/.github/workflows/build-kernel-deb.yml @@ -6,7 +6,7 @@ on: kernel-variant: description: 'Kernel variant identifier, naming this build in CI only' required: true - default: 'qcom-next' + default: 'qcom-next-trixie' type: string flavour: description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' @@ -421,7 +421,7 @@ jobs: - name: Upload prepared source tree artifact uses: actions/upload-artifact@v4 with: - name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.suite }} + name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} path: /tmp/kernel-srcpkg.tar.gz if-no-files-found: error retention-days: 7 @@ -436,7 +436,7 @@ jobs: uses: ./.github/workflows/build-kernel-debusine.yml with: suite: ${{ inputs.suite || 'trixie' }} - kernel-variant: ${{ inputs.kernel-variant || 'qcom-next' }} + flavour: ${{ inputs.flavour || 'qcom-next' }} srcpkg: ${{ inputs.srcpkg || 'linux-qcom-next' }} debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} target-workspace: ${{ inputs.target-workspace || '' }} @@ -455,4 +455,4 @@ jobs: uses: ./.github/workflows/build-kernel-ubuntu.yml with: distro: ${{ inputs.suite || 'resolute' }} - kernel-variant: ${{ inputs.kernel-variant || 'qcom-next' }} + flavour: ${{ inputs.flavour || 'qcom-next' }} diff --git a/.github/workflows/build-kernel-debusine.yml b/.github/workflows/build-kernel-debusine.yml index b632bf37..7b2bc193 100644 --- a/.github/workflows/build-kernel-debusine.yml +++ b/.github/workflows/build-kernel-debusine.yml @@ -28,8 +28,8 @@ on: description: 'Target Debian suite (trixie, forky, sid)' type: string required: true - kernel-variant: - description: 'Kernel variant identifier used to isolate artifacts and Debusine workspaces' + flavour: + description: 'Kernel flavour, isolating artifacts, Debusine workspaces and S3 paths' type: string required: true srcpkg: @@ -94,7 +94,7 @@ jobs: - name: Download prepared source tree uses: actions/download-artifact@v4 with: - name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.suite }} + name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} path: /tmp/srcpkg-artifact - name: Extract prepared source tree @@ -117,7 +117,7 @@ jobs: GITHUB_REPOSITORY_ID: ${{ github.repository_id }} GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - JOB_INDEX: ${{ inputs.kernel-variant }}-${{ inputs.suite }} + JOB_INDEX: ${{ inputs.flavour }}-${{ inputs.suite }} DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} @@ -200,7 +200,7 @@ jobs: with: s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} path: deb-artifacts - destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.kernel-variant }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ + destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ # --------------------------------------------------------------------------- # release: promote packages from CI workspace to target Debusine workspace. diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index f62317ec..97a15264 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -19,8 +19,8 @@ on: description: Target Ubuntu suite (noble, questing, resolute) type: string required: true - kernel-variant: - description: 'Kernel variant identifier used to isolate prepared-source artifacts and published packages' + flavour: + description: 'Kernel flavour, isolating prepared-source artifacts and published packages' type: string required: true @@ -60,7 +60,7 @@ jobs: - name: Download prepared source tree uses: actions/download-artifact@v4 with: - name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.distro }} + name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.distro }} path: /tmp/srcpkg-artifact - name: Extract prepared source tree @@ -96,4 +96,4 @@ jobs: with: s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} path: kernel-build/${{ inputs.distro }} - destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.kernel-variant }}/${{ inputs.distro }}/${{ github.run_id }}-${{ github.run_attempt }}/ + destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.distro }}/${{ github.run_id }}-${{ github.run_attempt }}/ diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index e5d7cf4b..b5747857 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -2,9 +2,8 @@ name: daily # Daily kernel package builds. Runs at 3:00 PM PST (23:00 UTC), one parallel # build per Daily entry in ci/build-matrix.yaml. Manual dispatch can run the -# full Daily matrix, every suite for one kernel variant, or one kernel variant -# and suite. Variant and suite are matrix values, so future variants need no -# workflow edit. +# full Daily matrix, a named set of variants, or every suite of one flavour. +# Variants and flavours are matrix values, so new ones need no workflow edit. # # Matrix schema: ci/build-matrix.yaml # One entry is one generated package, so ci/scripts/resolve-matrix.py selects @@ -24,23 +23,23 @@ on: workflow_dispatch: inputs: build-scope: - description: 'Choose all variants, all suites for one variant, or one variant and suite' + description: 'Choose the whole matrix, named build(s), or every suite of one flavour' type: choice required: true - default: 'Selected variant and suite' + default: 'Selected variants' options: - 'Full matrix' - - 'Selected variant (all suites)' - - 'Selected variant and suite' - kernel-variant: - description: 'Kernel variant to build (ignored for Full matrix)' + - 'Selected variants' + - 'Selected flavour (all suites)' + kernel-variants: + description: 'Comma-separated variants to build, e.g. qcom-next-trixie,qcom-next-debug-forky (Selected variants)' required: true - default: 'qcom-next' + default: 'qcom-next-trixie' type: string - suite: - description: 'Target suite to build (used only for Selected variant and suite)' + flavour: + description: 'Kernel flavour to build in every suite it targets (Selected flavour)' required: true - default: 'trixie' + default: 'qcom-next' type: string permissions: @@ -53,7 +52,8 @@ jobs: # # Delegates to ci/scripts/resolve-matrix.py, which validates the whole matrix # and returns the entries with type: Daily. Manual dispatch selects the - # complete matrix, every suite for a variant, or one variant and suite. + # complete matrix, a comma-separated list of variants, or one flavour across + # every suite it targets. # --------------------------------------------------------------------------- configure-matrix: name: Configure build matrix @@ -67,8 +67,8 @@ jobs: id: set-matrix env: BUILD_SCOPE_INPUT: ${{ github.event.inputs.build-scope }} - INPUT_VARIANT: ${{ github.event.inputs.kernel-variant }} - INPUT_SUITE: ${{ github.event.inputs.suite }} + INPUT_VARIANTS: ${{ github.event.inputs.kernel-variants }} + INPUT_FLAVOUR: ${{ github.event.inputs.flavour }} run: | set -euo pipefail @@ -83,13 +83,13 @@ jobs: echo "Source: ci/build-matrix.yaml (full Daily matrix)" args=(--type Daily) ;; - "Selected variant (all suites)") - echo "Source: ci/build-matrix.yaml (variant: $INPUT_VARIANT, all suites)" - args=(--type Daily --kernel-variant "$INPUT_VARIANT") + "Selected variants") + echo "Source: ci/build-matrix.yaml (variants: $INPUT_VARIANTS)" + args=(--type Daily --kernel-variant "$INPUT_VARIANTS") ;; - "Selected variant and suite") - echo "Source: ci/build-matrix.yaml (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" - args=(--type Daily --kernel-variant "$INPUT_VARIANT" --suite "$INPUT_SUITE") + "Selected flavour (all suites)") + echo "Source: ci/build-matrix.yaml (flavour: $INPUT_FLAVOUR, all suites)" + args=(--type Daily --flavour "$INPUT_FLAVOUR") ;; *) echo "ERROR: unsupported build scope: $BUILD_SCOPE" >&2 @@ -110,7 +110,7 @@ jobs: # build does not cancel the forky build. # --------------------------------------------------------------------------- build: - name: ${{ matrix.kernel_variant }} (${{ matrix.suite }}) + name: ${{ matrix.kernel_variant }} needs: configure-matrix strategy: fail-fast: false diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 4ef4263a..65943df8 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -52,7 +52,7 @@ jobs: # resolute results on the PR. # --------------------------------------------------------------------------- build: - name: ${{ matrix.kernel_variant }} (${{ matrix.suite }}) + name: ${{ matrix.kernel_variant }} needs: configure-matrix strategy: fail-fast: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 80707bc6..6df0c322 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,9 +8,9 @@ name: release # target workspace (e.g. qli) via debusine-action lib/release. # # To update a release ref: open a PR that changes branch_or_tag on every -# Release entry for that kernel variant in ci/build-matrix.yaml. A variant's -# Release entries must agree on the ref, so resolve-matrix.py rejects the -# matrix if only some of them are updated. +# Release entry for that flavour in ci/build-matrix.yaml. A flavour's Release +# entries must agree on the ref, so resolve-matrix.py rejects the matrix if +# only some of them are updated. # # This workflow intentionally has no schedule trigger. Releases are always # initiated manually after validation. @@ -19,22 +19,22 @@ on: workflow_dispatch: inputs: release-scope: - description: 'Choose all suites for one variant or one variant and suite' + description: 'Choose every suite of one flavour, or named build(s)' type: choice required: true - default: 'Selected variant (all suites)' + default: 'Selected flavour (all suites)' options: - - 'Selected variant (all suites)' - - 'Selected variant and suite' - kernel-variant: - description: 'Kernel variant to release' + - 'Selected flavour (all suites)' + - 'Selected variants' + flavour: + description: 'Kernel flavour to release in every suite it targets (Selected flavour)' required: true default: 'qcom-next' type: string - suite: - description: 'Target suite to release (used only for Selected variant and suite)' + kernel-variants: + description: 'Comma-separated variants to release, e.g. qcom-next-trixie,qcom-next-forky (Selected variants)' required: true - default: 'trixie' + default: 'qcom-next-trixie' type: string permissions: @@ -59,21 +59,21 @@ jobs: id: set-matrix env: RELEASE_SCOPE_INPUT: ${{ github.event.inputs.release-scope }} - INPUT_VARIANT: ${{ github.event.inputs.kernel-variant }} - INPUT_SUITE: ${{ github.event.inputs.suite }} + INPUT_FLAVOUR: ${{ github.event.inputs.flavour }} + INPUT_VARIANTS: ${{ github.event.inputs.kernel-variants }} run: | set -euo pipefail RELEASE_SCOPE="$RELEASE_SCOPE_INPUT" case "$RELEASE_SCOPE" in - "Selected variant (all suites)") - echo "Source: ci/build-matrix.yaml (variant: $INPUT_VARIANT, all suites)" - args=(--type Release --kernel-variant "$INPUT_VARIANT") + "Selected flavour (all suites)") + echo "Source: ci/build-matrix.yaml (flavour: $INPUT_FLAVOUR, all suites)" + args=(--type Release --flavour "$INPUT_FLAVOUR") ;; - "Selected variant and suite") - echo "Source: ci/build-matrix.yaml (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" - args=(--type Release --kernel-variant "$INPUT_VARIANT" --suite "$INPUT_SUITE") + "Selected variants") + echo "Source: ci/build-matrix.yaml (variants: $INPUT_VARIANTS)" + args=(--type Release --kernel-variant "$INPUT_VARIANTS") ;; *) echo "ERROR: unsupported release scope: $RELEASE_SCOPE" >&2 @@ -93,7 +93,7 @@ jobs: # release path (lib/release) instead of the S3 publish path. # --------------------------------------------------------------------------- build: - name: ${{ matrix.kernel_variant }} (${{ matrix.suite }}) + name: ${{ matrix.kernel_variant }} needs: configure-matrix strategy: fail-fast: false diff --git a/README.md b/README.md index 4c7b452a..ca9ab023 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,14 @@ Two entry points use the same reusable build pipeline: to the selected production Debusine workspace. One entry in `deliveries` is one generated package: a single `kernel_variant`, -a single `type`, and a single `suite`. Nothing is expanded or derived at -resolve time, so what an entry says is what gets built: +a single `type`, and a single `suite`. The `kernel_variant` names that one +build and nothing else — it is the Actions job name and what a manual dispatch +asks for. Nothing is expanded or derived at resolve time, so what an entry says +is what gets built: ```yaml deliveries: - - kernel_variant: qcom-next + - kernel_variant: qcom-next-trixie type: Daily suite: trixie flavour: qcom-next @@ -66,7 +68,7 @@ deliveries: - audioreach debian_revision: '0qli1~bpo13+1~' - - kernel_variant: qcom-next + - kernel_variant: qcom-next-trixie type: Release suite: trixie flavour: qcom-next @@ -132,8 +134,10 @@ Daily is the recurring build and artifact-publication path. - The scheduled run resolves the full `Daily` matrix. - A manual run selects one **Build scope**: - **Full matrix** builds every configured variant and suite. - - **Selected variant (all suites)** builds every configured suite for one variant. - - **Selected variant and suite** builds one isolated matrix leg. + - **Selected variants** builds a comma-separated list of `kernel_variant` + names, e.g. `qcom-next-trixie,qcom-next-debug-forky`. + - **Selected flavour (all suites)** builds one flavour in every suite it + targets. - `latest_tag` resolves the newest matching dated tag; `branch_tip` resolves the configured branch directly. - Debian suites build in Debusine, then their `.deb` outputs are downloaded and @@ -146,13 +150,13 @@ Daily is the recurring build and artifact-publication path. Release is the controlled promotion path. - It is manual only and uses one **Release scope** for a kernel variant: - - **Selected variant (all suites)** is the normal release action and promotes - every configured Release suite for that variant. - - **Selected variant and suite** promotes one configured Release suite for - that variant when a targeted action is required. + - **Selected flavour (all suites)** is the normal release action and promotes + every configured Release suite for that flavour. + - **Selected variants** promotes a comma-separated list of `kernel_variant` + names when a targeted action is required. - It uses the pinned `branch_or_tag` from the selected `Release` matrix row; it never resolves a newest tag. -- Debian source and binary artifacts are built in per-variant, per-suite +- Debian source and binary artifacts are built in per-flavour, per-suite Debusine CI workspaces. - Successful builds are promoted with Debusine's `package-publish` workflow to the `qli` workspace, where they are available through the production Debusine @@ -175,7 +179,7 @@ them to the workflow matrix as they stand. Each entry carries: | Field | Purpose | | --- | --- | -| `kernel_variant` | CI identifier for this build: its Actions job name, prepared-source artifact, Debusine child workspace and S3 path. Never reaches a package name or version. Lowercase letters, digits, and internal hyphens only. | +| `kernel_variant` | The name of this one build, and nothing else: its Actions job name, and what a manual dispatch asks for. Unique within a delivery type, so a Daily and its Release may share a name. Never reaches a package name, a version, or a published path. Lowercase letters, digits, and internal hyphens only. | | `type` | `Daily` or `Release`. | | `suite` | The one suite this entry builds for. | | `flavour` | The kernel's own identity, and the only matrix field that reaches the built kernel. `derive-localversion.sh` makes it the LOCALVERSION suffix, so two flavours built from one ref get distinct kernel releases and their `linux-image` packages coexist. All entries sharing a flavour build the same package for different suites. | @@ -206,8 +210,9 @@ Because entries are written out in full, the resolver also checks the invariants that span them, which is what makes the duplication safe to read at face value: -- No two entries share a `kernel_variant`, `type` and `suite` — one entry is - one generated package. +- No two entries of one `type` share a `kernel_variant` — a variant names + exactly one build, so a run cannot produce two jobs with one name and a + dispatch cannot be ambiguous. - A flavour's entries agree on `srcpkg`, `binpkg` and `kernel_config`; those decide what the package *is*, and the entries differ only in where it goes. - A flavour's entries for one suite agree on `dkms`. The module set depends on @@ -229,20 +234,21 @@ the variant and suite it was given (`resolve-matrix.py --field debian_revision`) and builds at the revision the daily build would have used. Each entry has a distinct prepared-source artifact, Debusine child workspace, -and S3 path keyed by `kernel_variant + suite`. This prevents two variants that -both build, for example, `trixie` from consuming or publishing each other's -inputs or outputs. +and S3 path keyed by `flavour + suite`. This prevents two flavours that both +build, for example, `trixie` from consuming or publishing each other's inputs +or outputs. They key on `flavour`, not `kernel_variant`, so renaming a build +leg never moves a published artifact. Daily S3 outputs use these layouts, where `` is `-`: ```text -/pkg/debusine///// -/pkg/temp///// +/pkg/debusine///// +/pkg/temp///// ``` The first layout is for Debian/Debusine builds; the second is for Ubuntu Docker -builds. Consumers must select the intended kernel variant and suite. +builds. Consumers must select the intended flavour and suite. Supporting scripts keep workflow YAML small and testable: @@ -452,7 +458,7 @@ The available inputs are: | Input | Default | Purpose | | --- | --- | --- | -| `kernel-variant` | `qcom-next` | CI identifier used for artifact and workspace identity only. | +| `kernel-variant` | `qcom-next-trixie` | Names this build; also selects the matrix entry a blank `debian-revision` is taken from. | | `flavour` | `qcom-next` | Kernel flavour: the LOCALVERSION suffix, and so the kernel release identity. | | `suite` | `trixie` | Target suite. | | `ref-strategy` | `latest_tag` | `latest_tag`, `branch_tip`, or `pinned_ref`. | @@ -501,7 +507,8 @@ To add a kernel variant: 1. Add one entry to `deliveries` per package the variant should produce: one per Daily suite and one per Release suite, each spelling out all of its own - fields. Do not rely on another variant's values. + fields, and each with a `kernel_variant` unique within its delivery type. + Do not rely on another variant's values. 2. Give them all the same `flavour`, distinct from every other flavour's — it becomes the LOCALVERSION suffix, so this is what lets the new kernel install alongside the existing ones. Keep `srcpkg`, `binpkg` and `kernel_config` diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index d4693b50..a16098b1 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -39,7 +39,7 @@ deliveries: # --------------------------------------------------------------------------- # qcom-next — the standard kernel. # --------------------------------------------------------------------------- - - kernel_variant: qcom-next + - kernel_variant: qcom-next-trixie type: Daily suite: trixie flavour: qcom-next @@ -57,7 +57,7 @@ deliveries: - audioreach debian_revision: '0qli1~bpo13+1~' - - kernel_variant: qcom-next + - kernel_variant: qcom-next-forky type: Daily suite: forky flavour: qcom-next @@ -75,7 +75,7 @@ deliveries: - audioreach debian_revision: '0qli1~' - - kernel_variant: qcom-next + - kernel_variant: qcom-next-resolute type: Daily suite: resolute flavour: qcom-next @@ -91,7 +91,7 @@ deliveries: - kgsl debian_revision: '0qli1~26.04.1~' - - kernel_variant: qcom-next + - kernel_variant: qcom-next-trixie type: Release suite: trixie flavour: qcom-next @@ -109,7 +109,7 @@ deliveries: debian_revision: '0qli1~bpo13+1' target_workspace: qli - - kernel_variant: qcom-next + - kernel_variant: qcom-next-forky type: Release suite: forky flavour: qcom-next @@ -131,7 +131,7 @@ deliveries: # qcom-next-debug — the same kernel ref built with the in-tree debug # fragments, packaged separately so it installs alongside qcom-next. # --------------------------------------------------------------------------- - - kernel_variant: qcom-next-debug + - kernel_variant: qcom-next-debug-trixie type: Daily suite: trixie flavour: qcom-next-debug @@ -151,7 +151,7 @@ deliveries: - audioreach debian_revision: '0qli1~bpo13+1~' - - kernel_variant: qcom-next-debug + - kernel_variant: qcom-next-debug-forky type: Daily suite: forky flavour: qcom-next-debug @@ -171,7 +171,7 @@ deliveries: - audioreach debian_revision: '0qli1~' - - kernel_variant: qcom-next-debug + - kernel_variant: qcom-next-debug-trixie type: Release suite: trixie flavour: qcom-next-debug @@ -191,7 +191,7 @@ deliveries: debian_revision: '0qli1~bpo13+1' target_workspace: qli - - kernel_variant: qcom-next-debug + - kernel_variant: qcom-next-debug-forky type: Release suite: forky flavour: qcom-next-debug diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index 9c7eed52..abe80be2 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -15,22 +15,32 @@ Usage: ci/scripts/resolve-matrix.py --type Daily - ci/scripts/resolve-matrix.py --type Release --kernel-variant qcom-next - ci/scripts/resolve-matrix.py --type Daily --kernel-variant qcom-next --suite trixie - ci/scripts/resolve-matrix.py --type Daily --kernel-variant qcom-next \\ - --suite trixie --field debian_revision + ci/scripts/resolve-matrix.py --type Daily --kernel-variant qcom-next-trixie + ci/scripts/resolve-matrix.py --type Daily --kernel-variant qcom-next-trixie,qcom-next-forky + ci/scripts/resolve-matrix.py --type Release --flavour qcom-next + ci/scripts/resolve-matrix.py --type Daily --kernel-variant qcom-next-trixie \\ + --field debian_revision Options: --type TYPE Delivery type to select (Daily or Release). Required. - --kernel-variant VARIANT Select only this kernel variant. - --suite SUITE Select only this suite. + --kernel-variant VARIANTS Select only these kernel variants, comma-separated. + A variant names one leg, so this is the way to + ask for a specific set of builds. + --flavour FLAVOURS Select only these flavours, comma-separated. A + flavour spans its suites, so this asks for one + kernel everywhere it is built. + --suite SUITES Select only these suites, comma-separated. --field NAME Print just this field of the single selected entry, unquoted. Errors unless exactly one entry matches. --matrix-file FILE Matrix path (default: ci/build-matrix.yaml relative to CWD). + Filters combine: an entry must match every filter given. Every name in a + filter must match at least one entry of the selected type, so a typo or a + stale variant fails instead of quietly narrowing the build set. + Output: Without --field, a compact JSON array of the selected entries, ready for a GitHub Actions matrix `include`. kernel_config and dkms are joined into the @@ -322,7 +332,7 @@ def check_consistency(deliveries, errors): by_flavour[flavour].append(entry) by_flavour_type[(flavour, delivery_type)].append(entry) by_flavour_suite[(flavour, suite)].append(entry) - by_leg[(entry.get("kernel_variant"), delivery_type, suite)].append(entry) + by_leg[(delivery_type, entry.get("kernel_variant"))].append(entry) for field in ("srcpkg", "binpkg"): if isinstance(entry.get(field), str) and entry[field]: @@ -333,16 +343,19 @@ def check_consistency(deliveries, errors): ): by_package_version[(entry["srcpkg"], entry["debian_revision"])].append(entry) - # kernel_variant plus suite is what names a build leg in the Actions UI and - # scopes its artifacts, workspace and S3 path, so a repeat would have two - # legs writing to one another's outputs. - for (variant, delivery_type, suite), entries in sorted( + # kernel_variant names one build leg: it is the Actions job name and what a + # workflow dispatch asks for by name. It only has to be unique within a + # delivery type, because a run only ever resolves one type, and that lets a + # Daily and its Release share a name. Two entries sharing both would give a + # run two identically named jobs and make the dispatch filter ambiguous. + for (delivery_type, variant), entries in sorted( by_leg.items(), key=lambda item: str(item[0]) ): if len(entries) > 1: + suites = ", ".join(sorted(str(e.get("suite")) for e in entries)) errors.append( - f"kernel_variant {variant} defines {len(entries)} {delivery_type} " - f"entries for suite {suite}; one entry is one generated package" + f"kernel_variant {variant} is used by {len(entries)} {delivery_type} " + f"entries (suites {suites}); a variant names exactly one build" ) for flavour, entries in sorted(by_flavour.items()): @@ -477,24 +490,56 @@ def load_matrix(path): return deliveries -def select(deliveries, delivery_type, kernel_variant, suite): - """Return the entries matching the requested filters, in matrix order.""" +def parse_filter(value): + """Split a comma-separated filter into names, or None when unset.""" + if not value: + return None + return [name.strip() for name in value.split(",") if name.strip()] + + +def select(deliveries, delivery_type, filters): + """Return the entries matching the requested filters, in matrix order. + + filters maps a field name to the list of values allowed for it, or to + None when that filter was not given. + """ return [ entry for entry in deliveries if entry["type"] == delivery_type - and kernel_variant in ("", entry["kernel_variant"]) - and suite in ("", entry["suite"]) + and all( + wanted is None or entry[field] in wanted + for field, wanted in filters.items() + ) ] -def describe_selection(delivery_type, kernel_variant, suite): +def unmatched_filters(deliveries, delivery_type, filters): + """Names asked for that no entry of this delivery type offers.""" + missing = [] + for field, wanted in filters.items(): + if wanted is None: + continue + available = { + entry[field] for entry in deliveries if entry["type"] == delivery_type + } + for name in wanted: + if name not in available: + missing.append( + f"no {delivery_type} entry has {field} {name} " + f"(available: {', '.join(sorted(available))})" + ) + return missing + + +def describe_selection(delivery_type, filters): """Render the active filters for an error message.""" parts = [f"type={delivery_type}"] - if kernel_variant: - parts.append(f"kernel_variant={kernel_variant}") - if suite: - parts.append(f"suite={suite}") + parts += [ + f"{field}={','.join(wanted)}" + for field, wanted in filters.items() + if wanted is not None + ] return " ".join(parts) @@ -516,9 +561,16 @@ def main(): "--type", required=True, choices=DELIVERY_TYPES, help="delivery type to select" ) parser.add_argument( - "--kernel-variant", default="", help="select only this kernel variant" + "--kernel-variant", + default="", + help="select only these kernel variants, comma-separated", + ) + parser.add_argument( + "--flavour", default="", help="select only these flavours, comma-separated" + ) + parser.add_argument( + "--suite", default="", help="select only these suites, comma-separated" ) - parser.add_argument("--suite", default="", help="select only this suite") parser.add_argument( "--field", default="", @@ -530,9 +582,24 @@ def main(): args = parser.parse_args() deliveries = load_matrix(args.matrix_file) - selected = select(deliveries, args.type, args.kernel_variant, args.suite) + filters = { + "kernel_variant": parse_filter(args.kernel_variant), + "flavour": parse_filter(args.flavour), + "suite": parse_filter(args.suite), + } + what = describe_selection(args.type, filters) + + # Report a name that matches nothing before reporting an empty selection: + # "no Daily entry has kernel_variant qcom-next" says what to fix, where + # "no entries found" leaves the reader to work out which filter was wrong. + unmatched = unmatched_filters(deliveries, args.type, filters) + if unmatched: + sys.exit( + f"ERROR: Nothing to build for {what}:\n" + + "\n".join(f" - {problem}" for problem in unmatched) + ) - what = describe_selection(args.type, args.kernel_variant, args.suite) + selected = select(deliveries, args.type, filters) if not selected: sys.exit(f"ERROR: No matrix entries found for {what}") From 1e1581656f136b6bc4d85cfdcde9c84b2dda1e75 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 18/71] ci: rename the matrix keys to builds and name The two keys were named for what the matrix used to be. "deliveries" described rows that were expanded into build legs, and "kernel_variant" identified a kernel variant that several rows shared. Neither is true any more: an entry is one build, and its identifier labels that one build and nothing else, having been separated from flavour and from every published path. Rename them to builds and name, which is what a YAML list of things each carrying an identifier normally looks like, and reads as such: builds: - name: qcom-next-trixie type: Daily suite: trixie flavour: qcom-next The resolver's filter follows: --kernel-variant becomes --build, and build-kernel-deb.yml's kernel-variant input becomes build. The dispatch inputs on daily.yml and release.yml become "builds", taking the same comma-separated list, and their scopes read "Selected builds". Both old spellings now fail rather than being ignored: deliveries is rejected as an unknown top-level key, and kernel_variant as an unknown field on an entry whose name is missing. Names only. No build changes what it produces: the job names, S3 paths, Debusine workspaces, revisions and LOCALVERSIONs are all as they were. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-deb.yml | 30 ++--- .github/workflows/build-kernel-debusine.yml | 2 +- .github/workflows/daily.yml | 32 +++--- .github/workflows/pr-build.yml | 4 +- .github/workflows/release.yml | 16 +-- README.md | 55 +++++----- ci/build-matrix.yaml | 31 +++--- ci/scripts/resolve-matrix.py | 116 ++++++++++---------- 8 files changed, 143 insertions(+), 143 deletions(-) diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml index 93c16fa3..07f6a668 100644 --- a/.github/workflows/build-kernel-deb.yml +++ b/.github/workflows/build-kernel-deb.yml @@ -3,8 +3,8 @@ name: build-kernel-deb on: workflow_dispatch: inputs: - kernel-variant: - description: 'Kernel variant identifier, naming this build in CI only' + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' required: true default: 'qcom-next-trixie' type: string @@ -86,10 +86,10 @@ on: # Called by daily.yml and release.yml as a reusable workflow. workflow_call: inputs: - kernel-variant: - description: 'Kernel variant identifier, naming this build in CI only' + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' type: string - default: 'qcom-next' + default: 'qcom-next-trixie' flavour: description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' type: string @@ -180,12 +180,12 @@ jobs: shell: bash env: SUITE_INPUT: ${{ inputs.suite || 'trixie' }} - KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} + BUILD_INPUT: ${{ inputs.build || 'qcom-next-trixie' }} run: | suite="$SUITE_INPUT" - kernel_variant="$KERNEL_VARIANT_INPUT" - [[ "$kernel_variant" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { - echo "ERROR: kernel variant must use lowercase letters, digits, and internal hyphens" >&2 + build_name="$BUILD_INPUT" + [[ "$build_name" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { + echo "ERROR: build name must use lowercase letters, digits, and internal hyphens" >&2 exit 1 } case "$suite" in @@ -246,7 +246,7 @@ jobs: BRANCH_INPUT: ${{ inputs.kernel-branch || 'qcom-next' }} REF_STRATEGY_INPUT: ${{ inputs.ref-strategy || 'latest_tag' }} TAG_PATTERN_INPUT: ${{ inputs.tag-pattern || 'qcom-next-*' }} - KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} + BUILD_INPUT: ${{ inputs.build || 'qcom-next-trixie' }} run: | set -euo pipefail BRANCH="$BRANCH_INPUT" @@ -269,7 +269,7 @@ jobs: esac echo "KERNEL_REF=$KERNEL_REF" >> "$GITHUB_ENV" - echo "Kernel variant: $KERNEL_VARIANT_INPUT" + echo "Build: $BUILD_INPUT" echo "Ref strategy: $REF_STRATEGY" echo "Resolved kernel ref: $KERNEL_REF" @@ -312,7 +312,7 @@ jobs: # direct dispatch is build-only and never promotes. env: DEBIAN_REVISION_INPUT: ${{ inputs.debian-revision }} - KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} + BUILD_INPUT: ${{ inputs.build || 'qcom-next-trixie' }} run: | set -euo pipefail INPUT_REVISION="$DEBIAN_REVISION_INPUT" @@ -322,14 +322,14 @@ jobs: else DEBIAN_REVISION=$(ci/scripts/resolve-matrix.py \ --type Daily \ - --kernel-variant "$KERNEL_VARIANT_INPUT" \ + --build "$BUILD_INPUT" \ --suite "$DISTRO" \ --field debian_revision) || { - echo "::error::No Daily matrix entry for kernel-variant=$KERNEL_VARIANT_INPUT suite=$DISTRO." + echo "::error::No Daily matrix entry named $BUILD_INPUT for suite $DISTRO." echo "::error::Add one to ci/build-matrix.yaml, or set the debian-revision input to build this combination ad hoc." exit 1 } - echo "Matrix Debian revision for $KERNEL_VARIANT_INPUT/$DISTRO: $DEBIAN_REVISION" + echo "Matrix Debian revision for $BUILD_INPUT/$DISTRO: $DEBIAN_REVISION" fi echo "DEBIAN_REVISION=$DEBIAN_REVISION" >> "$GITHUB_ENV" diff --git a/.github/workflows/build-kernel-debusine.yml b/.github/workflows/build-kernel-debusine.yml index 7b2bc193..d8ae9960 100644 --- a/.github/workflows/build-kernel-debusine.yml +++ b/.github/workflows/build-kernel-debusine.yml @@ -5,7 +5,7 @@ name: build-debusine # Called by build-kernel-deb.yml for Debian-family suites (trixie, forky, sid). # The kernel source is already prepared by the caller's prepare job and shared # as the kernel-srcpkg artifact. Artifact and workspace identity include both -# kernel_variant and suite so parallel variants cannot share inputs or outputs. +# flavour and suite so parallel flavours cannot share inputs or outputs. # # Publish path is determined by target-workspace: # target-workspace == '' -> daily path: download .deb files, upload to S3. diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index b5747857..06abc389 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -2,16 +2,16 @@ name: daily # Daily kernel package builds. Runs at 3:00 PM PST (23:00 UTC), one parallel # build per Daily entry in ci/build-matrix.yaml. Manual dispatch can run the -# full Daily matrix, a named set of variants, or every suite of one flavour. -# Variants and flavours are matrix values, so new ones need no workflow edit. +# full Daily matrix, a named set of builds, or every suite of one flavour. +# Build names and flavours are matrix values, so new ones need no workflow edit. # # Matrix schema: ci/build-matrix.yaml # One entry is one generated package, so ci/scripts/resolve-matrix.py selects # the entries with type: Daily and passes them through as they stand. Every -# field (kernel_variant, suite, flavour, srcpkg, binpkg, kernel_config, dkms, +# field (name, suite, flavour, srcpkg, binpkg, kernel_config, dkms, # debian_revision, git_clone, branch_or_tag, ref_strategy) becomes a -# build-kernel-deb.yml input. kernel_variant names the job and scopes its -# artifacts; flavour is what reaches the built kernel's version. +# build-kernel-deb.yml input. name labels the job; flavour is what reaches +# the built kernel's version and its published paths. # # To add a daily build target: add an entry to ci/build-matrix.yaml with # type: Daily. No workflow changes needed. @@ -26,13 +26,13 @@ on: description: 'Choose the whole matrix, named build(s), or every suite of one flavour' type: choice required: true - default: 'Selected variants' + default: 'Selected builds' options: - 'Full matrix' - - 'Selected variants' + - 'Selected builds' - 'Selected flavour (all suites)' - kernel-variants: - description: 'Comma-separated variants to build, e.g. qcom-next-trixie,qcom-next-debug-forky (Selected variants)' + builds: + description: 'Comma-separated build names, e.g. qcom-next-trixie,qcom-next-debug-forky (Selected builds)' required: true default: 'qcom-next-trixie' type: string @@ -52,8 +52,8 @@ jobs: # # Delegates to ci/scripts/resolve-matrix.py, which validates the whole matrix # and returns the entries with type: Daily. Manual dispatch selects the - # complete matrix, a comma-separated list of variants, or one flavour across - # every suite it targets. + # complete matrix, a comma-separated list of build names, or one flavour + # across every suite it targets. # --------------------------------------------------------------------------- configure-matrix: name: Configure build matrix @@ -67,7 +67,7 @@ jobs: id: set-matrix env: BUILD_SCOPE_INPUT: ${{ github.event.inputs.build-scope }} - INPUT_VARIANTS: ${{ github.event.inputs.kernel-variants }} + INPUT_BUILDS: ${{ github.event.inputs.builds }} INPUT_FLAVOUR: ${{ github.event.inputs.flavour }} run: | set -euo pipefail @@ -84,8 +84,8 @@ jobs: args=(--type Daily) ;; "Selected variants") - echo "Source: ci/build-matrix.yaml (variants: $INPUT_VARIANTS)" - args=(--type Daily --kernel-variant "$INPUT_VARIANTS") + echo "Source: ci/build-matrix.yaml (builds: $INPUT_BUILDS)" + args=(--type Daily --build "$INPUT_BUILDS") ;; "Selected flavour (all suites)") echo "Source: ci/build-matrix.yaml (flavour: $INPUT_FLAVOUR, all suites)" @@ -110,7 +110,7 @@ jobs: # build does not cancel the forky build. # --------------------------------------------------------------------------- build: - name: ${{ matrix.kernel_variant }} + name: ${{ matrix.name }} needs: configure-matrix strategy: fail-fast: false @@ -119,7 +119,7 @@ jobs: uses: ./.github/workflows/build-kernel-deb.yml with: suite: ${{ matrix.suite }} - kernel-variant: ${{ matrix.kernel_variant }} + build: ${{ matrix.name }} flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 65943df8..df03c318 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -52,7 +52,7 @@ jobs: # resolute results on the PR. # --------------------------------------------------------------------------- build: - name: ${{ matrix.kernel_variant }} + name: ${{ matrix.name }} needs: configure-matrix strategy: fail-fast: false @@ -61,7 +61,7 @@ jobs: uses: ./.github/workflows/build-kernel-deb.yml with: suite: ${{ matrix.suite }} - kernel-variant: ${{ matrix.kernel_variant }} + build: ${{ matrix.name }} flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6df0c322..e2e55206 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,14 +25,14 @@ on: default: 'Selected flavour (all suites)' options: - 'Selected flavour (all suites)' - - 'Selected variants' + - 'Selected builds' flavour: description: 'Kernel flavour to release in every suite it targets (Selected flavour)' required: true default: 'qcom-next' type: string - kernel-variants: - description: 'Comma-separated variants to release, e.g. qcom-next-trixie,qcom-next-forky (Selected variants)' + builds: + description: 'Comma-separated build names, e.g. qcom-next-trixie,qcom-next-forky (Selected builds)' required: true default: 'qcom-next-trixie' type: string @@ -60,7 +60,7 @@ jobs: env: RELEASE_SCOPE_INPUT: ${{ github.event.inputs.release-scope }} INPUT_FLAVOUR: ${{ github.event.inputs.flavour }} - INPUT_VARIANTS: ${{ github.event.inputs.kernel-variants }} + INPUT_BUILDS: ${{ github.event.inputs.builds }} run: | set -euo pipefail @@ -72,8 +72,8 @@ jobs: args=(--type Release --flavour "$INPUT_FLAVOUR") ;; "Selected variants") - echo "Source: ci/build-matrix.yaml (variants: $INPUT_VARIANTS)" - args=(--type Release --kernel-variant "$INPUT_VARIANTS") + echo "Source: ci/build-matrix.yaml (builds: $INPUT_BUILDS)" + args=(--type Release --build "$INPUT_BUILDS") ;; *) echo "ERROR: unsupported release scope: $RELEASE_SCOPE" >&2 @@ -93,7 +93,7 @@ jobs: # release path (lib/release) instead of the S3 publish path. # --------------------------------------------------------------------------- build: - name: ${{ matrix.kernel_variant }} + name: ${{ matrix.name }} needs: configure-matrix strategy: fail-fast: false @@ -102,7 +102,7 @@ jobs: uses: ./.github/workflows/build-kernel-deb.yml with: suite: ${{ matrix.suite }} - kernel-variant: ${{ matrix.kernel_variant }} + build: ${{ matrix.name }} flavour: ${{ matrix.flavour }} git-clone: ${{ matrix.git_clone }} kernel-branch: ${{ matrix.branch_or_tag }} diff --git a/README.md b/README.md index ca9ab023..3dacc10e 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,7 @@ into LOCALVERSION, so each produces a distinct kernel release (`+qcom-next--g` and `+qcom-next-debug--g`) and therefore a distinct versioned image package that can be installed alongside the other. See [docs/version.md](docs/version.md) for how the version strings are composed. -The flavour is what the kernel is; `kernel_variant` is only what CI calls the -build. +The flavour is what the kernel is; a build's `name` is only what CI calls it. `ci/build-matrix.yaml` is the source of truth; this table is a summary. @@ -42,15 +41,15 @@ Two entry points use the same reusable build pipeline: - **Release** uses a pinned matrix ref and promotes successful Debian packages to the selected production Debusine workspace. -One entry in `deliveries` is one generated package: a single `kernel_variant`, -a single `type`, and a single `suite`. The `kernel_variant` names that one -build and nothing else — it is the Actions job name and what a manual dispatch -asks for. Nothing is expanded or derived at resolve time, so what an entry says -is what gets built: +One entry in `builds` is one generated package: a single `name`, a single +`type`, and a single `suite`. The `name` labels that one build and nothing +else — it is the Actions job name and what a manual dispatch asks for. Nothing +is expanded or derived at resolve time, so what an entry says is what gets +built: ```yaml -deliveries: - - kernel_variant: qcom-next-trixie +builds: + - name: qcom-next-trixie type: Daily suite: trixie flavour: qcom-next @@ -68,7 +67,7 @@ deliveries: - audioreach debian_revision: '0qli1~bpo13+1~' - - kernel_variant: qcom-next-trixie + - name: qcom-next-trixie type: Release suite: trixie flavour: qcom-next @@ -134,8 +133,8 @@ Daily is the recurring build and artifact-publication path. - The scheduled run resolves the full `Daily` matrix. - A manual run selects one **Build scope**: - **Full matrix** builds every configured variant and suite. - - **Selected variants** builds a comma-separated list of `kernel_variant` - names, e.g. `qcom-next-trixie,qcom-next-debug-forky`. + - **Selected builds** builds a comma-separated list of build names, e.g. + `qcom-next-trixie,qcom-next-debug-forky`. - **Selected flavour (all suites)** builds one flavour in every suite it targets. - `latest_tag` resolves the newest matching dated tag; `branch_tip` resolves @@ -152,8 +151,8 @@ Release is the controlled promotion path. - It is manual only and uses one **Release scope** for a kernel variant: - **Selected flavour (all suites)** is the normal release action and promotes every configured Release suite for that flavour. - - **Selected variants** promotes a comma-separated list of `kernel_variant` - names when a targeted action is required. + - **Selected builds** promotes a comma-separated list of build names when a + targeted action is required. - It uses the pinned `branch_or_tag` from the selected `Release` matrix row; it never resolves a newest tag. - Debian source and binary artifacts are built in per-flavour, per-suite @@ -171,15 +170,15 @@ production release controls. ## Matrix Model -`ci/build-matrix.yaml` is a mapping with exactly one top-level key, -`deliveries`. One entry in it is one generated package, so there is no +`ci/build-matrix.yaml` is a mapping with exactly one top-level key, `builds`. +One entry in it is one generated package, so there is no expansion step: `ci/scripts/resolve-matrix.py` validates the whole document, selects the entries matching the requested type, variant, and suite, and hands them to the workflow matrix as they stand. Each entry carries: | Field | Purpose | | --- | --- | -| `kernel_variant` | The name of this one build, and nothing else: its Actions job name, and what a manual dispatch asks for. Unique within a delivery type, so a Daily and its Release may share a name. Never reaches a package name, a version, or a published path. Lowercase letters, digits, and internal hyphens only. | +| `name` | The name of this one build, and nothing else: its Actions job name, and what a manual dispatch asks for. Unique within a delivery type, so a Daily and its Release may share a name. Never reaches a package name, a version, or a published path. Lowercase letters, digits, and internal hyphens only. | | `type` | `Daily` or `Release`. | | `suite` | The one suite this entry builds for. | | `flavour` | The kernel's own identity, and the only matrix field that reaches the built kernel. `derive-localversion.sh` makes it the LOCALVERSION suffix, so two flavours built from one ref get distinct kernel releases and their `linux-image` packages coexist. All entries sharing a flavour build the same package for different suites. | @@ -210,9 +209,9 @@ Because entries are written out in full, the resolver also checks the invariants that span them, which is what makes the duplication safe to read at face value: -- No two entries of one `type` share a `kernel_variant` — a variant names - exactly one build, so a run cannot produce two jobs with one name and a - dispatch cannot be ambiguous. +- No two entries of one `type` share a `name` — a name identifies exactly one + build, so a run cannot produce two jobs with one name and a dispatch cannot + be ambiguous. - A flavour's entries agree on `srcpkg`, `binpkg` and `kernel_config`; those decide what the package *is*, and the entries differ only in where it goes. - A flavour's entries for one suite agree on `dkms`. The module set depends on @@ -236,8 +235,8 @@ debian_revision`) and builds at the revision the daily build would have used. Each entry has a distinct prepared-source artifact, Debusine child workspace, and S3 path keyed by `flavour + suite`. This prevents two flavours that both build, for example, `trixie` from consuming or publishing each other's inputs -or outputs. They key on `flavour`, not `kernel_variant`, so renaming a build -leg never moves a published artifact. +or outputs. They key on `flavour`, not on the build's `name`, so renaming a +build never moves a published artifact. Daily S3 outputs use these layouts, where `` is `-`: @@ -448,7 +447,7 @@ Use **Actions** → **build-kernel-deb** for a one-off build. It is an explicit override workflow, not a matrix-derived delivery flow: use `daily.yml` and `release.yml` for normal Daily and Release operations. -`kernel-variant`, `suite`, and `ref-strategy` are the required build selection. +`build`, `suite`, and `ref-strategy` are the required build selection. All remaining package, configuration, and PR inputs are advanced overrides for validation or debugging. Variant and suite are free-text matrix values rather than static dropdowns, so adding a matrix entry never requires editing the @@ -458,7 +457,7 @@ The available inputs are: | Input | Default | Purpose | | --- | --- | --- | -| `kernel-variant` | `qcom-next-trixie` | Names this build; also selects the matrix entry a blank `debian-revision` is taken from. | +| `build` | `qcom-next-trixie` | Names this build; also selects the matrix entry a blank `debian-revision` is taken from. | | `flavour` | `qcom-next` | Kernel flavour: the LOCALVERSION suffix, and so the kernel release identity. | | `suite` | `trixie` | Target suite. | | `ref-strategy` | `latest_tag` | `latest_tag`, `branch_tip`, or `pinned_ref`. | @@ -505,10 +504,10 @@ and keeps production approval controls in the workflow path. To add a kernel variant: -1. Add one entry to `deliveries` per package the variant should produce: one - per Daily suite and one per Release suite, each spelling out all of its own - fields, and each with a `kernel_variant` unique within its delivery type. - Do not rely on another variant's values. +1. Add one entry to `builds` per package the variant should produce: one per + Daily suite and one per Release suite, each spelling out all of its own + fields, and each with a `name` unique within its delivery type. Do not rely + on another variant's values. 2. Give them all the same `flavour`, distinct from every other flavour's — it becomes the LOCALVERSION suffix, so this is what lets the new kernel install alongside the existing ones. Keep `srcpkg`, `binpkg` and `kernel_config` diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index a16098b1..f9dcd4b9 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -3,8 +3,8 @@ # # Kernel delivery matrix. # -# One entry in "deliveries" is one generated package: a single kernel_variant, -# a single delivery type, and a single suite. The only list-valued fields are +# One entry in "builds" is one generated package: a single name, a single +# delivery type, and a single suite. The only list-valued fields are # kernel_config and dkms, which name the fragments and the out-of-tree modules # that go into that one package, so an entry needs no expansion — ci/scripts/ # resolve-matrix.py validates the document, filters it, and hands the matching @@ -21,9 +21,10 @@ # identity: it becomes the LOCALVERSION suffix, so qcom-next and # qcom-next-debug produce distinct kernel releases that install alongside each # other, and every suite building one flavour produces the same kernel release. -# kernel_variant names the build in CI - the Actions job, the prepared-source -# artifact, the Debusine workspace, the S3 path - and never reaches a package -# name or version. +# A build's name is what CI calls it: the Actions job name, and what a manual +# dispatch asks for by name. It never reaches a package name or version, and +# never a published path. It must be unique within a delivery type, so a Daily +# and the Release that supersedes it share one. # # dkms varies by suite because it depends on which -dkms packages the # target archive carries: the Debian suites bundle camx and iris-vpu, which @@ -35,11 +36,11 @@ # the ordering discussion in README.md). No two entries may produce the same # srcpkg at the same debian_revision. -deliveries: +builds: # --------------------------------------------------------------------------- # qcom-next — the standard kernel. # --------------------------------------------------------------------------- - - kernel_variant: qcom-next-trixie + - name: qcom-next-trixie type: Daily suite: trixie flavour: qcom-next @@ -57,7 +58,7 @@ deliveries: - audioreach debian_revision: '0qli1~bpo13+1~' - - kernel_variant: qcom-next-forky + - name: qcom-next-forky type: Daily suite: forky flavour: qcom-next @@ -75,7 +76,7 @@ deliveries: - audioreach debian_revision: '0qli1~' - - kernel_variant: qcom-next-resolute + - name: qcom-next-resolute type: Daily suite: resolute flavour: qcom-next @@ -91,7 +92,7 @@ deliveries: - kgsl debian_revision: '0qli1~26.04.1~' - - kernel_variant: qcom-next-trixie + - name: qcom-next-trixie type: Release suite: trixie flavour: qcom-next @@ -109,7 +110,7 @@ deliveries: debian_revision: '0qli1~bpo13+1' target_workspace: qli - - kernel_variant: qcom-next-forky + - name: qcom-next-forky type: Release suite: forky flavour: qcom-next @@ -131,7 +132,7 @@ deliveries: # qcom-next-debug — the same kernel ref built with the in-tree debug # fragments, packaged separately so it installs alongside qcom-next. # --------------------------------------------------------------------------- - - kernel_variant: qcom-next-debug-trixie + - name: qcom-next-debug-trixie type: Daily suite: trixie flavour: qcom-next-debug @@ -151,7 +152,7 @@ deliveries: - audioreach debian_revision: '0qli1~bpo13+1~' - - kernel_variant: qcom-next-debug-forky + - name: qcom-next-debug-forky type: Daily suite: forky flavour: qcom-next-debug @@ -171,7 +172,7 @@ deliveries: - audioreach debian_revision: '0qli1~' - - kernel_variant: qcom-next-debug-trixie + - name: qcom-next-debug-trixie type: Release suite: trixie flavour: qcom-next-debug @@ -191,7 +192,7 @@ deliveries: debian_revision: '0qli1~bpo13+1' target_workspace: qli - - kernel_variant: qcom-next-debug-forky + - name: qcom-next-debug-forky type: Release suite: forky flavour: qcom-next-debug diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index abe80be2..47d5cb90 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -3,8 +3,8 @@ # SPDX-License-Identifier: BSD-3-Clause-Clear """Validate and select entries from the kernel delivery matrix. -ci/build-matrix.yaml holds one entry per generated package: a single -kernel_variant, a single delivery type, and a single suite. Nothing here +ci/build-matrix.yaml holds one entry per generated package: a single named +build, one delivery type, and one suite. Nothing here expands or derives anything -- the matrix is already flat, and each entry states its own debian_revision. This script validates the whole document, selects the entries a caller asked for, and prints them. @@ -15,18 +15,18 @@ Usage: ci/scripts/resolve-matrix.py --type Daily - ci/scripts/resolve-matrix.py --type Daily --kernel-variant qcom-next-trixie - ci/scripts/resolve-matrix.py --type Daily --kernel-variant qcom-next-trixie,qcom-next-forky + ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie + ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie,qcom-next-forky ci/scripts/resolve-matrix.py --type Release --flavour qcom-next - ci/scripts/resolve-matrix.py --type Daily --kernel-variant qcom-next-trixie \\ + ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie \\ --field debian_revision Options: --type TYPE Delivery type to select (Daily or Release). Required. - --kernel-variant VARIANTS Select only these kernel variants, comma-separated. - A variant names one leg, so this is the way to - ask for a specific set of builds. + --build NAMES Select only these builds by name, comma-separated. + A name identifies one build, so this is the way + to ask for a specific set of them. --flavour FLAVOURS Select only these flavours, comma-separated. A flavour spans its suites, so this asks for one kernel everywhere it is built. @@ -39,7 +39,7 @@ Filters combine: an entry must match every filter given. Every name in a filter must match at least one entry of the selected type, so a typo or a - stale variant fails instead of quietly narrowing the build set. + stale name fails instead of quietly narrowing the build set. Output: Without --field, a compact JSON array of the selected entries, ready for a @@ -84,7 +84,7 @@ } REQUIRED_STRING_FIELDS = ( - "kernel_variant", + "name", "type", "suite", "flavour", @@ -108,12 +108,12 @@ REQUIRED_STRING_FIELDS + OPTIONAL_STRING_FIELDS + ("kernel_config", "dkms") ) -# Everything about what is built is anchored on flavour, not kernel_variant. -# The flavour is the kernel's own identity: it becomes the LOCALVERSION +# Everything about what is built is anchored on flavour, not on the build's +# name. The flavour is the kernel's own identity: it becomes the LOCALVERSION # suffix, so two flavours built from one ref produce distinct kernel releases -# whose linux-image packages install alongside each other. kernel_variant -# names the build in CI -- job names, artifacts, Debusine workspaces, S3 -# paths -- and carries no packaging meaning. +# whose linux-image packages install alongside each other. A build's name only +# labels it in CI -- the Actions job, and what a dispatch asks for -- and +# carries no packaging meaning. # Fields that identify the flavour itself rather than one of its build legs. # Every entry for a flavour must agree on them, because they decide what the @@ -240,7 +240,7 @@ def check_entry(entry, report): for field in sorted(set(entry) - KNOWN_FIELDS): report(f"unknown field {field}") - for field in ("kernel_variant", "suite", "flavour"): + for field in ("name", "suite", "flavour"): value = entry.get(field) if isinstance(value, str) and not NAME_RE.match(value): report(f"{field} must use lowercase letters, digits, and internal hyphens") @@ -296,21 +296,21 @@ def describe(entry, index): return f"entry {index}" parts = [ str(entry[field]) - for field in ("kernel_variant", "type", "suite") + for field in ("name", "type", "suite") if isinstance(entry.get(field), str) ] return f"entry {index} ({'/'.join(parts)})" if parts else f"entry {index}" -def check_consistency(deliveries, errors): +def check_consistency(builds, errors): """Validate the invariants that span entries. Entries are written out in full, so the matrix can state a flavour twice and disagree with itself. These checks are what makes that duplication safe to read at face value. - Everything about the package is grouped by flavour. kernel_variant is - only checked for the one thing it is used for: naming a build leg + Everything about the package is grouped by flavour. A build's name is + only checked for the one thing it is used for: identifying that build uniquely. """ by_leg = defaultdict(list) @@ -320,7 +320,7 @@ def check_consistency(deliveries, errors): by_package_version = defaultdict(list) flavours_by_package = defaultdict(set) - for entry in deliveries: + for entry in builds: if not isinstance(entry, dict): continue flavour = entry.get("flavour") @@ -332,7 +332,7 @@ def check_consistency(deliveries, errors): by_flavour[flavour].append(entry) by_flavour_type[(flavour, delivery_type)].append(entry) by_flavour_suite[(flavour, suite)].append(entry) - by_leg[(delivery_type, entry.get("kernel_variant"))].append(entry) + by_leg[(delivery_type, entry.get("name"))].append(entry) for field in ("srcpkg", "binpkg"): if isinstance(entry.get(field), str) and entry[field]: @@ -343,19 +343,19 @@ def check_consistency(deliveries, errors): ): by_package_version[(entry["srcpkg"], entry["debian_revision"])].append(entry) - # kernel_variant names one build leg: it is the Actions job name and what a - # workflow dispatch asks for by name. It only has to be unique within a - # delivery type, because a run only ever resolves one type, and that lets a - # Daily and its Release share a name. Two entries sharing both would give a - # run two identically named jobs and make the dispatch filter ambiguous. - for (delivery_type, variant), entries in sorted( + # A name identifies one build: it is the Actions job name and what a + # workflow dispatch asks for. It only has to be unique within a delivery + # type, because a run only ever resolves one type, and that lets a Daily and + # its Release share a name. Two entries sharing both would give a run two + # identically named jobs and make the dispatch filter ambiguous. + for (delivery_type, name), entries in sorted( by_leg.items(), key=lambda item: str(item[0]) ): if len(entries) > 1: suites = ", ".join(sorted(str(e.get("suite")) for e in entries)) errors.append( - f"kernel_variant {variant} is used by {len(entries)} {delivery_type} " - f"entries (suites {suites}); a variant names exactly one build" + f"name {name} is used by {len(entries)} {delivery_type} entries " + f"(suites {suites}); a name identifies exactly one build" ) for flavour, entries in sorted(by_flavour.items()): @@ -436,21 +436,21 @@ def check_consistency(deliveries, errors): ) -def validate(deliveries): +def validate(builds): """Return every problem found in the matrix, as a list of messages.""" errors = [] - for index, entry in enumerate(deliveries): + for index, entry in enumerate(builds): if not isinstance(entry, dict): errors.append(f"{describe(entry, index)}: delivery entries must be mappings") continue label = describe(entry, index) check_entry(entry, lambda message, label=label: errors.append(f"{label}: {message}")) - check_consistency(deliveries, errors) + check_consistency(builds, errors) return errors def load_matrix(path): - """Read, parse, and validate the matrix, returning its deliveries.""" + """Read, parse, and validate the matrix, returning its builds.""" try: with open(path, encoding="utf-8") as handle: document = yaml.safe_load(handle) @@ -462,32 +462,32 @@ def load_matrix(path): sys.exit(f"ERROR: Invalid YAML in {path}: {error}") if not isinstance(document, dict): - sys.exit(f"ERROR: {path}: matrix root must be a mapping with a deliveries key") + sys.exit(f"ERROR: {path}: matrix root must be a mapping with a builds key") - # deliveries is the whole schema. Anything else at the root is a leftover - # from an older matrix (suite_suffix_mapping, say) that would otherwise sit + # builds is the whole schema. Anything else at the root is a leftover from + # an older matrix (suite_suffix_mapping, say) that would otherwise sit # there looking authoritative while nothing read it. - unknown_root = sorted(set(document) - {"deliveries"}) + unknown_root = sorted(set(document) - {"builds"}) if unknown_root: sys.exit( f"ERROR: {path}: unknown top-level key(s) {', '.join(unknown_root)}; " - "deliveries is the only one" + "builds is the only one" ) - deliveries = document.get("deliveries") - if not isinstance(deliveries, list): - sys.exit(f"ERROR: {path}: deliveries must be a list") - if not deliveries: - sys.exit(f"ERROR: {path}: deliveries must contain at least one entry") + builds = document.get("builds") + if not isinstance(builds, list): + sys.exit(f"ERROR: {path}: builds must be a list") + if not builds: + sys.exit(f"ERROR: {path}: builds must contain at least one entry") - errors = validate(deliveries) + errors = validate(builds) if errors: sys.exit( f"ERROR: Invalid kernel delivery matrix in {path}:\n" + "\n".join(f" - {error}" for error in errors) ) - return deliveries + return builds def parse_filter(value): @@ -497,7 +497,7 @@ def parse_filter(value): return [name.strip() for name in value.split(",") if name.strip()] -def select(deliveries, delivery_type, filters): +def select(builds, delivery_type, filters): """Return the entries matching the requested filters, in matrix order. filters maps a field name to the list of values allowed for it, or to @@ -505,7 +505,7 @@ def select(deliveries, delivery_type, filters): """ return [ entry - for entry in deliveries + for entry in builds if entry["type"] == delivery_type and all( wanted is None or entry[field] in wanted @@ -514,14 +514,14 @@ def select(deliveries, delivery_type, filters): ] -def unmatched_filters(deliveries, delivery_type, filters): +def unmatched_filters(builds, delivery_type, filters): """Names asked for that no entry of this delivery type offers.""" missing = [] for field, wanted in filters.items(): if wanted is None: continue available = { - entry[field] for entry in deliveries if entry["type"] == delivery_type + entry[field] for entry in builds if entry["type"] == delivery_type } for name in wanted: if name not in available: @@ -561,9 +561,9 @@ def main(): "--type", required=True, choices=DELIVERY_TYPES, help="delivery type to select" ) parser.add_argument( - "--kernel-variant", + "--build", default="", - help="select only these kernel variants, comma-separated", + help="select only these builds by name, comma-separated", ) parser.add_argument( "--flavour", default="", help="select only these flavours, comma-separated" @@ -581,25 +581,25 @@ def main(): ) args = parser.parse_args() - deliveries = load_matrix(args.matrix_file) + builds = load_matrix(args.matrix_file) filters = { - "kernel_variant": parse_filter(args.kernel_variant), + "name": parse_filter(args.build), "flavour": parse_filter(args.flavour), "suite": parse_filter(args.suite), } what = describe_selection(args.type, filters) # Report a name that matches nothing before reporting an empty selection: - # "no Daily entry has kernel_variant qcom-next" says what to fix, where + # "no Daily entry has name qcom-next" says what to fix, where # "no entries found" leaves the reader to work out which filter was wrong. - unmatched = unmatched_filters(deliveries, args.type, filters) + unmatched = unmatched_filters(builds, args.type, filters) if unmatched: sys.exit( f"ERROR: Nothing to build for {what}:\n" + "\n".join(f" - {problem}" for problem in unmatched) ) - selected = select(deliveries, args.type, filters) + selected = select(builds, args.type, filters) if not selected: sys.exit(f"ERROR: No matrix entries found for {what}") @@ -610,7 +610,7 @@ def main(): if len(selected) > 1: sys.exit( f"ERROR: --field {args.field} needs exactly one entry, but {what} " - f"selects {len(selected)}; narrow it with --kernel-variant and --suite" + f"selects {len(selected)}; narrow it with --build" ) entry = for_workflow(selected[0]) From 7c330d3efa24105a373957675bee26f3758ea2bd Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 19/71] ci: add the mainline, linux-next and Arduino flavours Three kernels join the delivery matrix, each built for trixie and forky: mainline from the tip of Linus's tree, next from the newest next-YYYYMMDD tag of linux-next, and qcom-arduino from the early/hwe/arduino topic branch of kernel-topics. mainline and next give early warning of upstream breakage; qcom-arduino delivers hardware enablement that has not reached qcom-next. All three track something moving - two branch tips and a tag cut every night - so none of them has a Release entry: a Release must name an immutable ref, and neither a branch tip nor tonight's tag is one. They are built and published daily and never promoted to qli. That makes the "every flavour defines at least one of each delivery type" invariant wrong, so it becomes the half that carries the meaning: a Release entry requires the Daily that tests it, but a Daily needs no Release. The symmetric rule was not protecting anything - a flavour built daily and never promoted is a coherent thing to configure, and the rule only prevented saying so. mainline and next bundle no DKMS modules. kgsl, camx and iris-vpu are built against the Qualcomm tree, and a listed module is a presence contract: naming one whose BUILD_EXCLUSIVE gates exclude the kernel fails the build rather than shipping without it. qcom-arduino is a Qualcomm branch and carries the same three its qcom-next siblings do on the Debian suites. Signed-off-by: Christopher Obbard --- README.md | 23 +++++--- ci/build-matrix.yaml | 108 +++++++++++++++++++++++++++++++++++ ci/scripts/resolve-matrix.py | 14 +++-- 3 files changed, 132 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3dacc10e..eca38a6d 100644 --- a/README.md +++ b/README.md @@ -24,13 +24,20 @@ kernel flavour owns as many entries as it has suites and delivery types. |---------|----------------|-------------------|--------------|----------------|-------| | `qcom-next` | `linux-qcom-next` | `linux-image-qcom-next` | trixie, forky, resolute | trixie, forky | Standard kernel | | `qcom-next-debug` | `linux-qcom-next-debug` | `linux-image-qcom-next-debug` | trixie, forky | trixie, forky | Adds `arch/arm64/configs/qcom_debug.config` and `kernel/configs/debug.config` from the kernel source, via `intree:` entries | - -Both build the same kernel ref. `derive-localversion.sh` folds the *flavour* -into LOCALVERSION, so each produces a distinct kernel release -(`+qcom-next--g` and `+qcom-next-debug--g`) and therefore a -distinct versioned image package that can be installed alongside the other. See +| `qcom-arduino` | `linux-qcom-arduino` | `linux-image-qcom-arduino` | trixie, forky | none | Arduino hardware-enablement topic branch (`early/hwe/arduino` of `kernel-topics`) | +| `mainline` | `linux-mainline` | `linux-image-mainline` | trixie, forky | none | Tip of Linus's tree, tracked for early warning of upstream breakage. No DKMS modules | +| `next` | `linux-next` | `linux-image-next` | trixie, forky | none | Newest `next-YYYYMMDD` tag of linux-next. No DKMS modules | + +`derive-localversion.sh` folds the *flavour* into LOCALVERSION, so each +produces a distinct kernel release (`+qcom-next--g`, +`+qcom-next-debug--g`, and so on) and therefore a distinct versioned +image package that can be installed alongside the others. The flavour is what +the kernel is; a build's `name` is only what CI calls it. See [docs/version.md](docs/version.md) for how the version strings are composed. -The flavour is what the kernel is; a build's `name` is only what CI calls it. + +The last three track a moving upstream — a branch tip, or a tag cut every +night — and so have no `Release` entries: a release must name an immutable ref +to promote. They are built and published daily and never promoted to `qli`. `ci/build-matrix.yaml` is the source of truth; this table is a summary. @@ -221,7 +228,9 @@ face value: - A flavour's entries of one `type` agree on `git_clone`, `branch_or_tag`, `ref_strategy` and `tag_pattern`, so a forgotten suite cannot quietly ship a different kernel from its siblings after a release ref bump. -- Every flavour defines at least one `Daily` and one `Release` entry. +- A flavour with a `Release` entry also has a `Daily` one, so nothing is + promoted that the daily build has not tested. The converse is allowed: a + flavour tracking a moving upstream is built daily and never released. - No `srcpkg` or `binpkg` is shared between flavours, and no two entries build the same `srcpkg` at the same `debian_revision`. - Where a suite has both, its `Daily` revision is its `Release` revision plus a diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index f9dcd4b9..6c23b613 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -30,6 +30,11 @@ # target archive carries: the Debian suites bundle camx and iris-vpu, which # Ubuntu does not package. # +# A flavour tracking a moving upstream is Daily only. A Release entry must name +# an immutable ref, and a branch tip or a tag cut every night is not one, so +# mainline, next and the topic branches are built and published daily and never +# promoted. A Release entry still requires the Daily that tests it. +# # debian_revision is stated outright, not derived. Daily revisions end in "~" # so they sort below the Release revision for the same suite; the rest of the # revision places the suite where it belongs relative to the other suites (see @@ -211,3 +216,106 @@ builds: - audioreach debian_revision: '0qli1' target_workspace: qli + + # --------------------------------------------------------------------------- + # qcom-arduino — the Arduino hardware-enablement topic branch. + # + # Daily only: a topic branch has no immutable ref to promote from. + # --------------------------------------------------------------------------- + - name: qcom-arduino-trixie + type: Daily + suite: trixie + flavour: qcom-arduino + git_clone: https://github.com/qualcomm-linux/kernel-topics + branch_or_tag: early/hwe/arduino + ref_strategy: branch_tip + srcpkg: linux-qcom-arduino + binpkg: linux-image-qcom-arduino + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + debian_revision: '0qli~bpo13+1~' + + - name: qcom-arduino-forky + type: Daily + suite: forky + flavour: qcom-arduino + git_clone: https://github.com/qualcomm-linux/kernel-topics + branch_or_tag: early/hwe/arduino + ref_strategy: branch_tip + srcpkg: linux-qcom-arduino + binpkg: linux-image-qcom-arduino + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + debian_revision: '0qli~' + + # --------------------------------------------------------------------------- + # mainline — Linus's tree, tracked for early warning of upstream breakage. + # + # Daily only, and no DKMS: the out-of-tree modules target the Qualcomm tree, + # and a listed module is a presence contract that would fail the build here. + # --------------------------------------------------------------------------- + - name: mainline-trixie + type: Daily + suite: trixie + flavour: mainline + git_clone: https://github.com/torvalds/linux + branch_or_tag: master + ref_strategy: branch_tip + srcpkg: linux-mainline + binpkg: linux-image-mainline + kernel_config: [] + dkms: [] + debian_revision: '0qli~bpo13+1~' + + - name: mainline-forky + type: Daily + suite: forky + flavour: mainline + git_clone: https://github.com/torvalds/linux + branch_or_tag: master + ref_strategy: branch_tip + srcpkg: linux-mainline + binpkg: linux-image-mainline + kernel_config: [] + dkms: [] + debian_revision: '0qli~' + + # --------------------------------------------------------------------------- + # next — linux-next, the integration tree, one step ahead of mainline. + # + # next-YYYYMMDD tags are cut daily, so latest_tag resolves the newest of + # them the same way qcom-next-* does. Daily only, and no DKMS, as above. + # --------------------------------------------------------------------------- + - name: next-trixie + type: Daily + suite: trixie + flavour: next + git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git + branch_or_tag: master + ref_strategy: latest_tag + tag_pattern: 'next-*' + srcpkg: linux-next + binpkg: linux-image-next + kernel_config: [] + dkms: [] + debian_revision: '0qli~bpo13+1~' + + - name: next-forky + type: Daily + suite: forky + flavour: next + git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git + branch_or_tag: master + ref_strategy: latest_tag + tag_pattern: 'next-*' + srcpkg: linux-next + binpkg: linux-image-next + kernel_config: [] + dkms: [] + debian_revision: '0qli~' diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index 47d5cb90..3f5082ec 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -366,13 +366,15 @@ def check_consistency(builds, errors): f"flavour {flavour} must use one {field} across all its " "entries (got " + ", ".join(sorted(values)) + ")" ) + # A Release must be preceded by the Daily that tests it, but the + # converse does not hold: a flavour tracking a moving upstream has + # nothing immutable to pin, so it is built daily and never promoted. types = {entry.get("type") for entry in entries} - for delivery_type in DELIVERY_TYPES: - if delivery_type not in types: - errors.append( - f"flavour {flavour} has no {delivery_type} entry; " - "every flavour must define at least one of each" - ) + if "Release" in types and "Daily" not in types: + errors.append( + f"flavour {flavour} has a Release entry but no Daily entry; " + "a release must be preceded by the daily build that tests it" + ) for (flavour, suite), entries in sorted( by_flavour_suite.items(), key=lambda item: str(item[0]) From 2dc65b4c8167d5906fbd36bde14d3bc2754cee4f Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 20/71] ci: drop the resolve job and gate the build legs directly The resolve job took a whole ubuntu-latest runner to turn inputs.suite into the string "debian" or "ubuntu" via a static case statement, which the two build legs then compared against. A job-level if: expression answers the same question for free, so every run paid for a runner spin-up and a queue wait ahead of both legs to learn something already fixed at dispatch time. Both legs now test the suite themselves, the Ubuntu one negating what the Debian one asserts, so the two cannot drift into an overlap or a gap. The list is spelled out twice because GitHub Actions does not expand YAML anchors and there is no other way to share it without reintroducing a job. The trixie default is unchanged. One behaviour change comes with it: GitHub's contains() compares strings case-insensitively, so a suite of "Trixie" now classifies as debian where the case statement sent it to ubuntu. Neither spelling reaches here from daily.yml or release.yml, whose suites come from the matrix in lowercase. The build-name check moves to the Resolve Debian revision step, the only place the name is consumed, and becomes an ::error:: so it shows in the run summary rather than only in the log. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-deb.yml | 43 +++++--------------------- 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml index 07f6a668..184e5896 100644 --- a/.github/workflows/build-kernel-deb.yml +++ b/.github/workflows/build-kernel-deb.yml @@ -166,37 +166,6 @@ permissions: packages: read jobs: - # --------------------------------------------------------------------------- - # resolve: classify the target suite as 'debian' or 'ubuntu'. - # --------------------------------------------------------------------------- - resolve: - name: Resolve suite family - runs-on: ubuntu-latest - outputs: - family: ${{ steps.classify.outputs.family }} - steps: - - name: Classify suite - id: classify - shell: bash - env: - SUITE_INPUT: ${{ inputs.suite || 'trixie' }} - BUILD_INPUT: ${{ inputs.build || 'qcom-next-trixie' }} - run: | - suite="$SUITE_INPUT" - build_name="$BUILD_INPUT" - [[ "$build_name" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { - echo "ERROR: build name must use lowercase letters, digits, and internal hyphens" >&2 - exit 1 - } - case "$suite" in - trixie|forky|sid|unstable|bookworm) - family=debian ;; - *) - family=ubuntu ;; - esac - echo "family=$family" >> "$GITHUB_OUTPUT" - echo "Suite '$suite' classified as family=$family" - # --------------------------------------------------------------------------- # prepare: clone kernel source, inject packaging, run prepare-source.sh. # --------------------------------------------------------------------------- @@ -315,6 +284,10 @@ jobs: BUILD_INPUT: ${{ inputs.build || 'qcom-next-trixie' }} run: | set -euo pipefail + [[ "$BUILD_INPUT" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { + echo "::error::build name must use lowercase letters, digits, and internal hyphens" + exit 1 + } INPUT_REVISION="$DEBIAN_REVISION_INPUT" if [[ -n "$INPUT_REVISION" ]]; then DEBIAN_REVISION="$INPUT_REVISION" @@ -431,8 +404,8 @@ jobs: # --------------------------------------------------------------------------- debusine-build: name: Build in Debusine - needs: [resolve, prepare] - if: ${{ needs.resolve.outputs.family == 'debian' }} + needs: [prepare] + if: ${{ contains(fromJSON('["trixie", "forky", "sid", "unstable", "bookworm"]'), inputs.suite || 'trixie') }} uses: ./.github/workflows/build-kernel-debusine.yml with: suite: ${{ inputs.suite || 'trixie' }} @@ -450,8 +423,8 @@ jobs: # --------------------------------------------------------------------------- ubuntu-build: name: Build (Ubuntu, docker) - needs: [resolve, prepare] - if: ${{ needs.resolve.outputs.family == 'ubuntu' }} + needs: [prepare] + if: ${{ !contains(fromJSON('["trixie", "forky", "sid", "unstable", "bookworm"]'), inputs.suite || 'trixie') }} uses: ./.github/workflows/build-kernel-ubuntu.yml with: distro: ${{ inputs.suite || 'resolute' }} From 386c56d9b8703100fd8deaa075fa46dd89430820 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 21/71] ci: branch on the build path in the caller, not inside the build A full daily run rendered 21 skipped jobs against 11 real build legs, and a PR the same. Every leg started both family backends and skipped one; every Debian leg started both Debusine tails and skipped one. Release was worse per leg: all four of its legs are Debian, so the Ubuntu backend never once ran. Both branches were decided by data the caller already knew. A suite's family follows from the suite, which the matrix states; publish-versus-promote is fixed by which workflow is running, since a daily never promotes and a release always does. Passing both down into one shared workflow and re-deciding them there with if: is what minted the skipped jobs, and GitHub offers no way to take that back: a job is conditional or it is not, an empty dynamic matrix is an error rather than an empty job list, and the guard that avoids the error is another skipped job. So decide in the caller and give each decision its own workflow. resolve-matrix.py derives a family from each entry's suite and filters on it, daily.yml and pr-build.yml split one selection into a debian and an ubuntu matrix, and each leg calls only the workflow that builds it. build-kernel-deb.yml, which existed to make the choice, is gone. The two Debian workflows are the same build with different tails -- build-kernel-debian.yml publishes to S3, release-kernel-debian.yml promotes to a target workspace -- so neither carries the other's tail as a job to skip. Release entries only ever reach the second: promotion runs through Debusine, which builds no Ubuntu suite, so resolve-matrix.py now rejects a Release entry for one. That was previously accepted and would have built the package, taken the Ubuntu path, published to the daily S3 location and reported success without releasing anything. The shared steps move into two composite actions rather than a common parent workflow. A reusable workflow would have put its name into every job's, which is the nesting that "qcom-next-resolute / Build (Ubuntu, docker) / Build kernel package" already suffered from; a composite action adds no segment. That name is now "qcom-next-resolute / Build kernel package", and a full daily run is 32 jobs across 11 legs with nothing skipped. Suite stays a free-text dispatch input, so a new suite still needs no workflow edit. Since the workflow now implies the family, prepare-kernel-source takes the family its caller builds and rejects a suite belonging to the other before anything is cloned. Its family list is the one resolve-matrix.py routes by. Two dispatch scopes were repaired in passing: daily.yml and release.yml both still matched "Selected variants" in a case whose input has offered "Selected builds" since the matrix keys were renamed, so choosing it failed the run with "unsupported build scope". No build changes what it produces. The job names, S3 paths, Debusine workspaces, revisions and LOCALVERSIONs are all as they were. The staging build-dependency workspaces come along in the same move. They were picked by a ternary on target-workspace, which was the only thing separating a daily run from a release one while both lived in one workflow. That separation is now the choice of workflow, so debusine-build takes the list as an input and each caller states its own: build-kernel-debian.yml reads qli and qli-staging, release-kernel-debian.yml reads qli alone. Same two answers, no longer derived from a field that no longer distinguishes anything. Signed-off-by: Christopher Obbard --- .github/actions/debusine-build/action.yml | 125 +++++ .../actions/prepare-kernel-source/action.yml | 318 +++++++++++++ .github/workflows/build-kernel-deb.yml | 431 ------------------ .github/workflows/build-kernel-debian.yml | 328 +++++++++++++ .github/workflows/build-kernel-debusine.yml | 248 ---------- .github/workflows/build-kernel-ubuntu.yml | 234 ++++++++-- .github/workflows/daily.yml | 77 +++- .github/workflows/pr-build.yml | 59 ++- .github/workflows/release-kernel-debian.yml | 210 +++++++++ .github/workflows/release.yml | 14 +- README.md | 119 +++-- ci/scripts/resolve-matrix.py | 77 +++- 12 files changed, 1461 insertions(+), 779 deletions(-) create mode 100644 .github/actions/debusine-build/action.yml create mode 100644 .github/actions/prepare-kernel-source/action.yml delete mode 100644 .github/workflows/build-kernel-deb.yml create mode 100644 .github/workflows/build-kernel-debian.yml delete mode 100644 .github/workflows/build-kernel-debusine.yml create mode 100644 .github/workflows/release-kernel-debian.yml diff --git a/.github/actions/debusine-build/action.yml b/.github/actions/debusine-build/action.yml new file mode 100644 index 00000000..0eff3ade --- /dev/null +++ b/.github/actions/debusine-build/action.yml @@ -0,0 +1,125 @@ +name: Build in Debusine +description: > + Generate the Debian source package from the prepared tree and submit it to + Debusine, returning the CI workspace it was built in. + +# Shared by the two Debian workflows, which differ only in what they do with +# the result: build-kernel-debian.yml downloads the .deb files and uploads them +# to S3, release-kernel-debian.yml promotes them to a target workspace. Holding +# these steps in a composite action lets each workflow declare its own runner, +# container and environment while keeping one copy of the build itself. +# +# The caller must have checked out pkg-linux-qcom into the workspace first, +# which is also where this action comes from. Runs inside the +# debusine-pkg-builder container. +# +# vars and secrets contexts do not reach a composite action, so the Debusine +# host, scope and credentials come in as inputs from the calling workflow. + +inputs: + suite: + description: 'Target Debian suite (trixie, forky, sid)' + required: true + flavour: + description: 'Kernel flavour, isolating artifacts and Debusine workspaces' + required: true + debusine-host: + description: 'Debusine instance hostname' + required: true + debusine-scope: + description: 'Debusine scope' + required: true + debusine-user: + description: 'Debusine account used to submit the build' + required: true + debusine-token: + description: 'Debusine token for the CI child workspace' + required: true + debusine-parent-workspace: + description: 'Parent Debusine workspace for CI child workspace creation' + default: 'qli-ci' + extra-build-dep-workspaces: + description: > + Workspaces the build resolves its build-dependencies from, space + separated. A staging build may depend on a package version that has only + reached qli-staging; a release build reads qli alone, so a released + kernel is never built against something unreleased. + default: 'qli' + debusine-action-ref: + description: 'Ref of qualcomm-linux/debusine-action to use' + default: 'main' + +outputs: + workspace: + description: 'Debusine CI workspace the package was built in' + value: ${{ steps.build-debusine.outputs.workspace }} + workspace_url: + description: 'Browser URL of that workspace' + value: ${{ steps.build-debusine.outputs.workspace_url }} + srcpkg_version: + description: 'Version of the generated source package' + value: ${{ steps.generate-srcpkg.outputs.srcpkg_version }} + +runs: + using: composite + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@v5 + with: + repository: qualcomm-linux/debusine-action + ref: ${{ inputs.debusine-action-ref }} + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + - name: Download prepared source tree + uses: actions/download-artifact@v4 + with: + name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} + path: /tmp/srcpkg-artifact + + - name: Extract prepared source tree + shell: bash + run: | + mkdir srcpkg + tar xzf /tmp/srcpkg-artifact/*.tar.gz \ + -C srcpkg --strip-components=1 + + - name: Generate source package + id: generate-srcpkg + shell: bash + env: + SUITE: ${{ inputs.suite }} + DEBUSINE_ASSEMBLE_ORIG: "true" + run: | + SUITE="$SUITE" debusine-action/lib/generate-source-package + + - name: Build in Debusine + id: build-debusine + shell: bash + env: + GITHUB_REPOSITORY_ID: ${{ github.repository_id }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + JOB_INDEX: ${{ inputs.flavour }}-${{ inputs.suite }} + DEBUSINE_HOST: ${{ inputs.debusine-host }} + DEBUSINE_SCOPE: ${{ inputs.debusine-scope }} + DEBUSINE_USER: ${{ inputs.debusine-user }} + DEBUSINE_TOKEN: ${{ inputs.debusine-token }} + DEBUSINE_PARENT_WORKSPACE: ${{ inputs.debusine-parent-workspace }} + SUITE: ${{ inputs.suite }} + EXTRA_BUILD_DEP_WORKSPACES: ${{ inputs.extra-build-dep-workspaces }} + run: | + # No -x here: DEBUSINE_TOKEN is in env, keep xtrace off. + set -euo pipefail + debusine-action/lib/build + workspace=$(sed -n 's/^workspace=//p' "$GITHUB_OUTPUT") + echo "workspace_url=https://${DEBUSINE_HOST}/${DEBUSINE_SCOPE}/${workspace}/" >> "$GITHUB_OUTPUT" + + - name: Note Debusine workspace URL + shell: bash + env: + WORKSPACE_URL: ${{ steps.build-debusine.outputs.workspace_url }} + run: | + echo "Debusine Workspace URL: $WORKSPACE_URL" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml new file mode 100644 index 00000000..ae171847 --- /dev/null +++ b/.github/actions/prepare-kernel-source/action.yml @@ -0,0 +1,318 @@ +name: Prepare kernel source +description: > + Resolve the kernel ref, clone it, apply any PR patches, inject debian/ and + generate the source package, then upload the prepared tree as an artifact. + +# Every build takes this path, whichever family it belongs to, so it lives in a +# composite action rather than a reusable workflow: the two family workflows +# call it as a step of their own prepare job, which keeps one copy of these +# steps without adding a nesting level to every job name in the checks list. +# +# The caller must have checked out pkg-linux-qcom into the workspace first: +# debian/ and ci/ come from that checkout, and so does this action. +# +# Runs on the self-hosted arm64 runner. Leaves KERNEL_DIR, KERNEL_REF, +# KERNEL_SHA, LOCALVERSION and DEBIAN_REVISION in the job environment. + +inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + required: true + suite: + description: 'Target suite' + required: true + expect-family: + description: > + Build family the calling workflow builds, debian or ubuntu. The suite is + a free-text matrix value, so this catches one handed to the workflow for + the other family before anything is cloned or built. + required: true + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + required: true + kernel-url: + description: 'Kernel repository URL' + default: 'https://github.com/qualcomm-linux/kernel' + ref-strategy: + description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' + default: 'latest_tag' + kernel-branch: + description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' + default: 'qcom-next' + tag-pattern: + description: 'Tag glob used only by latest_tag' + default: 'qcom-next-*' + srcpkg: + description: 'Debian source package name' + default: 'linux-qcom-next' + binpkg: + description: 'Kernel image metapackage name' + default: 'linux-image-qcom-next' + kernel-config: + description: 'Extra config fragments applied on top of all of debian/config-available/' + default: '' + dkms: + description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' + default: '' + debian-revision: + description: 'Debian revision override (empty takes the Daily revision the matrix gives this build)' + default: '' + localversion: + description: 'LOCALVERSION override (auto-derived from the resolved ref if empty)' + default: '' + kver-extra: + description: 'Extra package-version suffix (e.g. -ci42)' + default: '' + debug-build: + description: 'Enable the debug build configuration' + default: 'false' + qcom-next-pr: + description: 'qcom-next PR numbers to merge, space-separated' + default: '' + kernel-topics-pr: + description: 'kernel-topics PR numbers to apply as patches, space-separated' + default: '' + +runs: + using: composite + steps: + - name: Resolve build environment + shell: bash + env: + SUITE_INPUT: ${{ inputs.suite }} + EXPECT_FAMILY: ${{ inputs.expect-family }} + run: | + set -euo pipefail + # Family and docker image in one place, because they answer the same + # question about a suite. Debian-family suites build in the trixie + # image: docker-pkg-build and pkg-builder only support trixie for + # Debian, and Debusine handles the actual suite-specific build + # internally. Ubuntu-family suites build in an image of their own + # suite. The family list here is the one in + # ci/scripts/resolve-matrix.py, which routes an entry to the workflow + # that calls this action. + case "$SUITE_INPUT" in + trixie|bookworm) FAMILY=debian; DOCKER_DISTRO="$SUITE_INPUT" ;; + forky|sid|unstable) FAMILY=debian; DOCKER_DISTRO=trixie ;; + *) FAMILY=ubuntu; DOCKER_DISTRO="$SUITE_INPUT" ;; + esac + + if [[ "$FAMILY" != "$EXPECT_FAMILY" ]]; then + echo "::error::Suite $SUITE_INPUT is $FAMILY-family, but this workflow builds the $EXPECT_FAMILY family." + echo "::error::Dispatch the $FAMILY build workflow for it instead." + exit 1 + fi + + echo "DISTRO=$SUITE_INPUT" >> "$GITHUB_ENV" + echo "DOCKER_DISTRO=$DOCKER_DISTRO" >> "$GITHUB_ENV" + echo "Suite $SUITE_INPUT is $FAMILY-family and builds in the $DOCKER_DISTRO image" + + - name: Checkout docker-pkg-build + uses: actions/checkout@v4 + with: + repository: qualcomm-linux/docker-pkg-build + ref: main + path: docker-pkg-build + + - name: Build docker image + shell: bash + run: | + ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DOCKER_DISTRO" + docker image ls + + - name: Resolve kernel ref + shell: bash + env: + KERNEL_URL_INPUT: ${{ inputs.kernel-url }} + BRANCH_INPUT: ${{ inputs.kernel-branch }} + REF_STRATEGY_INPUT: ${{ inputs.ref-strategy }} + TAG_PATTERN_INPUT: ${{ inputs.tag-pattern }} + BUILD_INPUT: ${{ inputs.build }} + run: | + set -euo pipefail + [[ "$BUILD_INPUT" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { + echo "::error::build name must use lowercase letters, digits, and internal hyphens" + exit 1 + } + echo "KERNEL_URL=$KERNEL_URL_INPUT" >> "$GITHUB_ENV" + + case "$REF_STRATEGY_INPUT" in + latest_tag) + KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh \ + --url "$KERNEL_URL_INPUT" \ + --latest-tag "$TAG_PATTERN_INPUT") + ;; + branch_tip|pinned_ref) + KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh \ + --url "$KERNEL_URL_INPUT" --ref "$BRANCH_INPUT") + ;; + *) + echo "::error::unsupported ref strategy: $REF_STRATEGY_INPUT" + exit 1 + ;; + esac + + echo "KERNEL_REF=$KERNEL_REF" >> "$GITHUB_ENV" + echo "Build: $BUILD_INPUT" + echo "Ref strategy: $REF_STRATEGY_INPUT" + echo "Resolved kernel ref: $KERNEL_REF" + + - name: Clone kernel source + shell: bash + run: | + set -euo pipefail + KERNEL_DIR="$GITHUB_WORKSPACE/kernel-source" + echo "KERNEL_DIR=$KERNEL_DIR" >> "$GITHUB_ENV" + echo "Cloning $KERNEL_URL @ $KERNEL_REF (shallow)..." + git clone --depth 1 --single-branch --branch "$KERNEL_REF" --no-tags \ + "$KERNEL_URL" "$KERNEL_DIR" + KERNEL_SHA=$(git -C "$KERNEL_DIR" rev-parse HEAD) + echo "KERNEL_SHA=$KERNEL_SHA" >> "$GITHUB_ENV" + echo "Kernel HEAD SHA: $KERNEL_SHA" + + - name: Derive LOCALVERSION + shell: bash + env: + LOCALVERSION_INPUT: ${{ inputs.localversion }} + FLAVOUR_INPUT: ${{ inputs.flavour }} + run: | + set -euo pipefail + if [[ -n "$LOCALVERSION_INPUT" ]]; then + echo "Using explicit LOCALVERSION override: $LOCALVERSION_INPUT" + echo "LOCALVERSION=$LOCALVERSION_INPUT" >> "$GITHUB_ENV" + else + LOCALVERSION=$(ci/scripts/derive-localversion.sh \ + --flavour "$FLAVOUR_INPUT" \ + --ref "$KERNEL_REF" \ + --sha "$(echo "$KERNEL_SHA" | cut -c1-12)") + echo "LOCALVERSION=$LOCALVERSION" >> "$GITHUB_ENV" + echo "Derived LOCALVERSION: $LOCALVERSION" + fi + + - name: Resolve Debian revision + # Callers driven by the matrix pass the entry's own debian-revision + # straight through. A direct workflow_dispatch that leaves it empty looks + # the same entry up itself, so a manual build of a configured build gets + # exactly the revision the daily build would have used. The lookup is + # always the Daily entry: direct dispatch is build-only and never + # promotes. + shell: bash + env: + DEBIAN_REVISION_INPUT: ${{ inputs.debian-revision }} + BUILD_INPUT: ${{ inputs.build }} + run: | + set -euo pipefail + if [[ -n "$DEBIAN_REVISION_INPUT" ]]; then + DEBIAN_REVISION="$DEBIAN_REVISION_INPUT" + echo "Using supplied Debian revision: $DEBIAN_REVISION" + else + DEBIAN_REVISION=$(ci/scripts/resolve-matrix.py \ + --type Daily \ + --build "$BUILD_INPUT" \ + --suite "$DISTRO" \ + --field debian_revision) || { + echo "::error::No Daily matrix entry named $BUILD_INPUT for suite $DISTRO." + echo "::error::Add one to ci/build-matrix.yaml, or set the debian-revision input to build this combination ad hoc." + exit 1 + } + echo "Matrix Debian revision for $BUILD_INPUT/$DISTRO: $DEBIAN_REVISION" + fi + echo "DEBIAN_REVISION=$DEBIAN_REVISION" >> "$GITHUB_ENV" + + - name: Merge qcom-next PR patches + if: ${{ inputs.qcom-next-pr != '' }} + shell: bash + env: + QCOM_NEXT_PR: ${{ inputs.qcom-next-pr }} + run: | + cd "$KERNEL_DIR" + echo "Merging qcom-next PR(s): $QCOM_NEXT_PR" + for pr in $QCOM_NEXT_PR; do + [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid qcom-next PR number: $pr"; exit 1; } + echo "::group::Merging qcom-next PR #$pr" + git fetch --no-tags origin "pull/$pr/head:pr-$pr" + if ! git merge --no-ff --no-commit "pr-$pr"; then + echo "ERROR: Merge conflict while merging PR #$pr. Aborting." + git merge --abort || true + exit 1 + fi + if ! git diff --cached --quiet; then + git commit -m "Merged qcom-next PR #$pr" + echo "PR #$pr merged successfully." + else + echo "PR #$pr already present; nothing to merge." + fi + echo "::endgroup::" + done + + - name: Apply kernel-topics PR patches + if: ${{ inputs.kernel-topics-pr != '' }} + shell: bash + env: + KERNEL_TOPICS_PR: ${{ inputs.kernel-topics-pr }} + run: | + cd "$KERNEL_DIR" + echo "Applying kernel-topics PR patch(es): $KERNEL_TOPICS_PR" + for pr in $KERNEL_TOPICS_PR; do + [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid kernel-topics PR number: $pr"; exit 1; } + echo "::group::Applying kernel-topics PR #$pr" + wget -q "https://github.com/qualcomm-linux/kernel-topics/pull/$pr.patch" -O "$pr.patch" + if ! git am "$pr.patch"; then + echo "ERROR: Patch application failed for PR #$pr. Aborting." + git am --abort || true + exit 1 + fi + echo "PR #$pr applied successfully." + echo "::endgroup::" + done + + - name: Prepare source + shell: bash + env: + SRCPKG_INPUT: ${{ inputs.srcpkg }} + BINPKG_INPUT: ${{ inputs.binpkg }} + KVER_EXTRA_INPUT: ${{ inputs.kver-extra }} + KERNEL_CONFIG_INPUT: ${{ inputs.kernel-config }} + DKMS_INPUT: ${{ inputs.dkms }} + DEBUG_BUILD_INPUT: ${{ inputs.debug-build }} + run: | + # No -x here: keep xtrace off for this step as it assembles the + # prepare-source.sh argument list which may include sensitive paths. + set -euo pipefail + + ARGS=( + --source-dir "$KERNEL_DIR" + --distro "$DISTRO" + --srcpkg "$SRCPKG_INPUT" + --binpkg "$BINPKG_INPUT" + --debian-revision "$DEBIAN_REVISION" + ) + + [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") + [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") + [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") + # Always passed, unlike the optional inputs above: an empty list is a + # genuine "bundle nothing" rather than a request for some default. + ARGS+=(--dkms "$DKMS_INPUT") + [[ "$DEBUG_BUILD_INPUT" == "true" ]] && ARGS+=(--debug) + + docker run -i --rm \ + -v "$PWD:$PWD" --workdir="$PWD" \ + "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" \ + ./prepare-source.sh "${ARGS[@]}" + + - name: Pack prepared source tree + shell: bash + run: | + set -euo pipefail + tar czf /tmp/kernel-srcpkg.tar.gz \ + -C "$(dirname "$KERNEL_DIR")" \ + "$(basename "$KERNEL_DIR")" + + - name: Upload prepared source tree artifact + uses: actions/upload-artifact@v4 + with: + name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} + path: /tmp/kernel-srcpkg.tar.gz + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml deleted file mode 100644 index 184e5896..00000000 --- a/.github/workflows/build-kernel-deb.yml +++ /dev/null @@ -1,431 +0,0 @@ -name: build-kernel-deb - -on: - workflow_dispatch: - inputs: - build: - description: 'Name of this build, as listed in ci/build-matrix.yaml' - required: true - default: 'qcom-next-trixie' - type: string - flavour: - description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' - required: true - default: 'qcom-next' - type: string - suite: - description: 'Target suite' - required: true - default: 'trixie' - type: string - ref-strategy: - description: 'Kernel ref strategy' - required: true - default: 'latest_tag' - type: choice - options: - - latest_tag - - branch_tip - - pinned_ref - kernel-branch: - description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' - required: false - default: 'qcom-next' - tag-pattern: - description: 'Tag glob used only by latest_tag (ignored for branch_tip and pinned_ref)' - required: false - default: 'qcom-next-*' - kernel-url: - description: 'Advanced: custom kernel repository URL (empty uses qualcomm-linux/kernel)' - required: false - default: '' - srcpkg: - description: 'Advanced: Debian source package name' - required: false - default: 'linux-qcom-next' - binpkg: - description: 'Advanced: kernel image metapackage name' - required: false - default: 'linux-image-qcom-next' - kernel-config: - description: 'Advanced: extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' - required: false - default: '' - dkms: - description: 'Advanced: out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' - required: false - default: '' - debian-revision: - description: 'Advanced: Debian revision override (empty takes the Daily revision the matrix gives this variant and suite)' - required: false - default: '' - localversion: - description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' - required: false - default: '' - kver-extra: - description: 'Advanced: extra package-version suffix (e.g. -ci42)' - required: false - default: '' - debug-build: - description: 'Advanced: enable the debug build configuration' - type: boolean - required: false - default: false - qcom-next-pr: - description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' - type: string - required: false - default: '' - kernel-topics-pr: - description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' - type: string - required: false - default: '' - - # Called by daily.yml and release.yml as a reusable workflow. - workflow_call: - inputs: - build: - description: 'Name of this build, as listed in ci/build-matrix.yaml' - type: string - default: 'qcom-next-trixie' - flavour: - description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' - type: string - default: 'qcom-next' - suite: - description: 'Target suite (trixie, forky, noble, questing, resolute, sid)' - type: string - default: 'trixie' - kernel-branch: - description: 'Source branch or pinned tag used by branch_tip or pinned_ref' - type: string - default: 'qcom-next' - git-clone: - description: 'Kernel repository URL' - type: string - default: 'https://github.com/qualcomm-linux/kernel' - ref-strategy: - description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' - type: string - default: 'latest_tag' - tag-pattern: - description: 'Tag glob used when ref-strategy=latest_tag' - type: string - default: 'qcom-next-*' - srcpkg: - description: 'Source package name' - type: string - default: 'linux-qcom-next' - binpkg: - description: 'Binary metapackage name' - type: string - default: 'linux-image-qcom-next' - kernel-config: - description: 'Extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' - type: string - default: '' - dkms: - description: 'Out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' - type: string - default: '' - debian-revision: - description: 'Debian revision component of the package version' - type: string - default: '0qli~' - localversion: - description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' - type: string - default: '' - kver-extra: - description: 'Extra suffix appended to the package version' - type: string - default: '' - debusine-parent-workspace: - description: 'Parent Debusine workspace for CI child workspace creation' - type: string - default: '' - target-workspace: - description: 'Debusine target workspace for release promotion (empty = daily/S3 path)' - type: string - default: '' - secrets: - DEBUSINE_USER: - description: 'Debusine account used to submit CI and release builds' - required: false - DEBUSINE_TOKEN: - description: 'Debusine token for the CI child workspace' - required: false - DEBUSINE_RELEASE_TOKEN: - description: 'Debusine token for package-publish promotion to the release target workspace' - required: false - -permissions: - contents: read - packages: read - -jobs: - # --------------------------------------------------------------------------- - # prepare: clone kernel source, inject packaging, run prepare-source.sh. - # --------------------------------------------------------------------------- - prepare: - name: Prepare kernel source - runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] - - env: - KERNEL_URL: ${{ inputs.git-clone || inputs.kernel-url || 'https://github.com/qualcomm-linux/kernel' }} - DISTRO: ${{ inputs.suite || 'trixie' }} - # For Debian-family suites (trixie, forky, sid) use trixie as the docker - # image distro: docker-pkg-build and pkg-builder only support trixie for - # Debian. Debusine handles the actual suite-specific build internally. - # For Ubuntu-family suites (noble, questing, resolute) use the real suite. - DOCKER_DISTRO: ${{ (inputs.suite == 'forky' || inputs.suite == 'sid' || inputs.suite == 'unstable') && 'trixie' || inputs.suite || 'trixie' }} - - defaults: - run: - shell: bash - - steps: - - name: Checkout pkg-linux-qcom - # debian/ and ci/ live on the same branch, so one checkout supplies - # both the packaging and resolve-kernel-ref.sh, derive-localversion.sh, - # resolve-matrix.py and the delivery matrix it reads. - # - # An empty ref leaves actions/checkout on github.sha, the commit this - # run was dispatched from or the one the calling workflow runs at. - # Credentials are kept: the self-pr step below fetches from origin. - uses: actions/checkout@v4 - - - name: Checkout docker-pkg-build - uses: actions/checkout@v4 - with: - repository: qualcomm-linux/docker-pkg-build - ref: main - path: docker-pkg-build - - - name: Build docker image - run: | - ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DOCKER_DISTRO" - docker image ls - - - name: Resolve kernel ref - id: kernel-ref - env: - BRANCH_INPUT: ${{ inputs.kernel-branch || 'qcom-next' }} - REF_STRATEGY_INPUT: ${{ inputs.ref-strategy || 'latest_tag' }} - TAG_PATTERN_INPUT: ${{ inputs.tag-pattern || 'qcom-next-*' }} - BUILD_INPUT: ${{ inputs.build || 'qcom-next-trixie' }} - run: | - set -euo pipefail - BRANCH="$BRANCH_INPUT" - REF_STRATEGY="$REF_STRATEGY_INPUT" - TAG_PATTERN="$TAG_PATTERN_INPUT" - - case "$REF_STRATEGY" in - latest_tag) - KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh \ - --url "$KERNEL_URL" \ - --latest-tag "$TAG_PATTERN") - ;; - branch_tip|pinned_ref) - KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh --url "$KERNEL_URL" --ref "$BRANCH") - ;; - *) - echo "ERROR: unsupported ref strategy: $REF_STRATEGY" >&2 - exit 1 - ;; - esac - - echo "KERNEL_REF=$KERNEL_REF" >> "$GITHUB_ENV" - echo "Build: $BUILD_INPUT" - echo "Ref strategy: $REF_STRATEGY" - echo "Resolved kernel ref: $KERNEL_REF" - - - name: Clone kernel source - run: | - KERNEL_DIR="$GITHUB_WORKSPACE/kernel-source" - echo "KERNEL_DIR=$KERNEL_DIR" >> "$GITHUB_ENV" - echo "Cloning $KERNEL_URL @ $KERNEL_REF (shallow)..." - git clone --depth 1 --single-branch --branch "$KERNEL_REF" --no-tags \ - "$KERNEL_URL" "$KERNEL_DIR" - KERNEL_SHA=$(git -C "$KERNEL_DIR" rev-parse HEAD) - echo "KERNEL_SHA=$KERNEL_SHA" >> "$GITHUB_ENV" - echo "Kernel HEAD SHA: $KERNEL_SHA" - - - name: Derive LOCALVERSION - env: - LOCALVERSION_INPUT: ${{ inputs.localversion }} - FLAVOUR_INPUT: ${{ inputs.flavour || 'qcom-next' }} - run: | - set -euo pipefail - OVERRIDE_LV="$LOCALVERSION_INPUT" - if [[ -n "$OVERRIDE_LV" ]]; then - echo "Using explicit LOCALVERSION override: $OVERRIDE_LV" - echo "LOCALVERSION=$OVERRIDE_LV" >> "$GITHUB_ENV" - else - LOCALVERSION=$(ci/scripts/derive-localversion.sh \ - --flavour "$FLAVOUR_INPUT" \ - --ref "$KERNEL_REF" \ - --sha "$(echo "$KERNEL_SHA" | cut -c1-12)") - echo "LOCALVERSION=$LOCALVERSION" >> "$GITHUB_ENV" - echo "Derived LOCALVERSION: $LOCALVERSION" - fi - - - name: Resolve Debian revision - # workflow_call callers (daily.yml, release.yml) pass the entry's own - # debian-revision straight from the matrix. A direct workflow_dispatch - # that leaves it empty looks the same entry up itself, so a manual - # build of a configured variant and suite gets exactly the revision the - # daily build would have used. The lookup is always the Daily entry: - # direct dispatch is build-only and never promotes. - env: - DEBIAN_REVISION_INPUT: ${{ inputs.debian-revision }} - BUILD_INPUT: ${{ inputs.build || 'qcom-next-trixie' }} - run: | - set -euo pipefail - [[ "$BUILD_INPUT" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { - echo "::error::build name must use lowercase letters, digits, and internal hyphens" - exit 1 - } - INPUT_REVISION="$DEBIAN_REVISION_INPUT" - if [[ -n "$INPUT_REVISION" ]]; then - DEBIAN_REVISION="$INPUT_REVISION" - echo "Using supplied Debian revision: $DEBIAN_REVISION" - else - DEBIAN_REVISION=$(ci/scripts/resolve-matrix.py \ - --type Daily \ - --build "$BUILD_INPUT" \ - --suite "$DISTRO" \ - --field debian_revision) || { - echo "::error::No Daily matrix entry named $BUILD_INPUT for suite $DISTRO." - echo "::error::Add one to ci/build-matrix.yaml, or set the debian-revision input to build this combination ad hoc." - exit 1 - } - echo "Matrix Debian revision for $BUILD_INPUT/$DISTRO: $DEBIAN_REVISION" - fi - echo "DEBIAN_REVISION=$DEBIAN_REVISION" >> "$GITHUB_ENV" - - - name: Merge qcom-next PR patches - if: ${{ inputs.qcom-next-pr != '' }} - env: - QCOM_NEXT_PR: ${{ inputs.qcom-next-pr }} - run: | - cd "$KERNEL_DIR" - echo "Merging qcom-next PR(s): $QCOM_NEXT_PR" - for pr in $QCOM_NEXT_PR; do - [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid qcom-next PR number: $pr"; exit 1; } - echo "::group::Merging qcom-next PR #$pr" - git fetch --no-tags origin "pull/$pr/head:pr-$pr" - if ! git merge --no-ff --no-commit "pr-$pr"; then - echo "ERROR: Merge conflict while merging PR #$pr. Aborting." - git merge --abort || true - exit 1 - fi - if ! git diff --cached --quiet; then - git commit -m "Merged qcom-next PR #$pr" - echo "PR #$pr merged successfully." - else - echo "PR #$pr already present; nothing to merge." - fi - echo "::endgroup::" - done - - - name: Apply kernel-topics PR patches - if: ${{ inputs.kernel-topics-pr != '' }} - env: - KERNEL_TOPICS_PR: ${{ inputs.kernel-topics-pr }} - run: | - cd "$KERNEL_DIR" - echo "Applying kernel-topics PR patch(es): $KERNEL_TOPICS_PR" - for pr in $KERNEL_TOPICS_PR; do - [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid kernel-topics PR number: $pr"; exit 1; } - echo "::group::Applying kernel-topics PR #$pr" - wget -q "https://github.com/qualcomm-linux/kernel-topics/pull/$pr.patch" -O "$pr.patch" - if ! git am "$pr.patch"; then - echo "ERROR: Patch application failed for PR #$pr. Aborting." - git am --abort || true - exit 1 - fi - echo "PR #$pr applied successfully." - echo "::endgroup::" - done - - - name: Prepare source - env: - SRCPKG_INPUT: ${{ inputs.srcpkg || 'linux-qcom-next' }} - BINPKG_INPUT: ${{ inputs.binpkg || 'linux-image-qcom-next' }} - KVER_EXTRA_INPUT: ${{ inputs.kver-extra }} - KERNEL_CONFIG_INPUT: ${{ inputs.kernel-config }} - DKMS_INPUT: ${{ inputs.dkms }} - DEBUG_BUILD_INPUT: ${{ inputs.debug-build }} - run: | - # No -x here: keep xtrace off for this step as it assembles the - # prepare-source.sh argument list which may include sensitive paths. - set -euo pipefail - - ARGS=( - --source-dir "$KERNEL_DIR" - --distro "$DISTRO" - --srcpkg "$SRCPKG_INPUT" - --binpkg "$BINPKG_INPUT" - --debian-revision "$DEBIAN_REVISION" - ) - - [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") - [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") - [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") - # Always passed, unlike the optional inputs above: an empty list is a - # genuine "bundle nothing" rather than a request for some default. - ARGS+=(--dkms "$DKMS_INPUT") - [[ "$DEBUG_BUILD_INPUT" == "true" ]] && ARGS+=(--debug) - - docker run -i --rm \ - -v "$PWD:$PWD" --workdir="$PWD" \ - "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" \ - ./prepare-source.sh "${ARGS[@]}" - - - name: Upload prepared source tree - run: | - tar czf /tmp/kernel-srcpkg.tar.gz \ - -C "$(dirname "$KERNEL_DIR")" \ - "$(basename "$KERNEL_DIR")" - - - name: Upload prepared source tree artifact - uses: actions/upload-artifact@v4 - with: - name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} - path: /tmp/kernel-srcpkg.tar.gz - if-no-files-found: error - retention-days: 7 - - # --------------------------------------------------------------------------- - # debusine-build: Debian suites only (trixie, forky, sid). - # --------------------------------------------------------------------------- - debusine-build: - name: Build in Debusine - needs: [prepare] - if: ${{ contains(fromJSON('["trixie", "forky", "sid", "unstable", "bookworm"]'), inputs.suite || 'trixie') }} - uses: ./.github/workflows/build-kernel-debusine.yml - with: - suite: ${{ inputs.suite || 'trixie' }} - flavour: ${{ inputs.flavour || 'qcom-next' }} - srcpkg: ${{ inputs.srcpkg || 'linux-qcom-next' }} - debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} - target-workspace: ${{ inputs.target-workspace || '' }} - secrets: - DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} - - # --------------------------------------------------------------------------- - # ubuntu-build: Ubuntu suites only (noble, questing, resolute). - # --------------------------------------------------------------------------- - ubuntu-build: - name: Build (Ubuntu, docker) - needs: [prepare] - if: ${{ !contains(fromJSON('["trixie", "forky", "sid", "unstable", "bookworm"]'), inputs.suite || 'trixie') }} - uses: ./.github/workflows/build-kernel-ubuntu.yml - with: - distro: ${{ inputs.suite || 'resolute' }} - flavour: ${{ inputs.flavour || 'qcom-next' }} diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml new file mode 100644 index 00000000..a885bcea --- /dev/null +++ b/.github/workflows/build-kernel-debian.yml @@ -0,0 +1,328 @@ +name: build-kernel-debian + +# Build a kernel package for a Debian-family suite and publish it to S3. +# +# Called per matrix entry by daily.yml and pr-build.yml, which select the +# Debian entries with resolve-matrix.py --family debian and call only this +# workflow for them. Nothing here is conditional on the suite: a caller that +# reaches this file has already decided the build belongs on the Debusine +# path, so every job in it runs. +# +# release-kernel-debian.yml is the same build with the release tail: it +# promotes to a target Debusine workspace instead of publishing to S3. +# +# Jobs: +# prepare (self-hosted): clone, patch, inject debian/, generate the source. +# build (debusine-pkg-builder container): submit to Debusine. +# publish (self-hosted): download the .deb files and upload them to S3. + +on: + workflow_call: + inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + type: string + default: 'qcom-next-trixie' + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + type: string + default: 'qcom-next' + suite: + description: 'Target Debian suite (trixie, forky, sid)' + type: string + default: 'trixie' + kernel-branch: + description: 'Source branch or pinned tag used by branch_tip or pinned_ref' + type: string + default: 'qcom-next' + git-clone: + description: 'Kernel repository URL' + type: string + default: 'https://github.com/qualcomm-linux/kernel' + ref-strategy: + description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' + type: string + default: 'latest_tag' + tag-pattern: + description: 'Tag glob used when ref-strategy=latest_tag' + type: string + default: 'qcom-next-*' + srcpkg: + description: 'Source package name' + type: string + default: 'linux-qcom-next' + binpkg: + description: 'Binary metapackage name' + type: string + default: 'linux-image-qcom-next' + kernel-config: + description: 'Extra config fragments applied on top of all of debian/config-available/' + type: string + default: '' + dkms: + description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' + type: string + default: '' + debian-revision: + description: 'Debian revision component of the package version' + type: string + default: '0qli~' + localversion: + description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' + type: string + default: '' + kver-extra: + description: 'Extra suffix appended to the package version' + type: string + default: '' + debug-build: + description: 'Enable the debug build configuration' + type: boolean + default: false + debusine-parent-workspace: + description: 'Parent Debusine workspace for CI child workspace creation' + type: string + default: '' + qcom-next-pr: + description: 'qcom-next PR numbers to merge (e.g. "42 43")' + type: string + default: '' + kernel-topics-pr: + description: 'kernel-topics PR numbers to apply as patches' + type: string + default: '' + secrets: + DEBUSINE_USER: + description: 'Debusine account used to submit CI builds' + required: false + DEBUSINE_TOKEN: + description: 'Debusine token for the CI child workspace' + required: false + + # Ad-hoc build of one Debian suite. Release promotion is not offered here: + # a manual build never promotes, so it always takes the S3 publish path. + workflow_dispatch: + inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + required: true + default: 'qcom-next-trixie' + type: string + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + required: true + default: 'qcom-next' + type: string + suite: + description: 'Target Debian suite (trixie, forky, sid). An Ubuntu suite here is rejected before anything is built; dispatch build-kernel-ubuntu for those.' + required: true + default: 'trixie' + type: string + ref-strategy: + description: 'Kernel ref strategy' + required: true + default: 'latest_tag' + type: choice + options: + - latest_tag + - branch_tip + - pinned_ref + kernel-branch: + description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' + required: false + default: 'qcom-next' + tag-pattern: + description: 'Tag glob used only by latest_tag (ignored for branch_tip and pinned_ref)' + required: false + default: 'qcom-next-*' + git-clone: + description: 'Advanced: custom kernel repository URL' + required: false + default: 'https://github.com/qualcomm-linux/kernel' + srcpkg: + description: 'Advanced: Debian source package name' + required: false + default: 'linux-qcom-next' + binpkg: + description: 'Advanced: kernel image metapackage name' + required: false + default: 'linux-image-qcom-next' + kernel-config: + description: 'Advanced: extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' + required: false + default: '' + dkms: + description: 'Advanced: out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' + required: false + default: '' + debian-revision: + description: 'Advanced: Debian revision override (empty takes the Daily revision the matrix gives this build and suite)' + required: false + default: '' + localversion: + description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' + required: false + default: '' + kver-extra: + description: 'Advanced: extra package-version suffix (e.g. -ci42)' + required: false + default: '' + debug-build: + description: 'Advanced: enable the debug build configuration' + type: boolean + required: false + default: false + qcom-next-pr: + description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' + type: string + required: false + default: '' + kernel-topics-pr: + description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' + type: string + required: false + default: '' + +permissions: + contents: read + packages: read + +jobs: + # --------------------------------------------------------------------------- + # prepare: clone kernel source, inject packaging, run prepare-source.sh. + # --------------------------------------------------------------------------- + prepare: + name: Prepare kernel source + runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + steps: + # debian/ and ci/ live on the same branch, so one checkout supplies both + # the packaging and the scripts, the delivery matrix they read, and the + # prepare-kernel-source action below. + # + # An empty ref leaves actions/checkout on github.sha, the commit this run + # was dispatched from or the one the calling workflow runs at. + - name: Checkout pkg-linux-qcom + uses: actions/checkout@v4 + + - name: Prepare kernel source + uses: ./.github/actions/prepare-kernel-source + with: + build: ${{ inputs.build }} + suite: ${{ inputs.suite }} + expect-family: debian + flavour: ${{ inputs.flavour }} + kernel-url: ${{ inputs.git-clone }} + ref-strategy: ${{ inputs.ref-strategy }} + kernel-branch: ${{ inputs.kernel-branch }} + tag-pattern: ${{ inputs.tag-pattern }} + srcpkg: ${{ inputs.srcpkg }} + binpkg: ${{ inputs.binpkg }} + kernel-config: ${{ inputs.kernel-config }} + dkms: ${{ inputs.dkms }} + debian-revision: ${{ inputs.debian-revision }} + localversion: ${{ inputs.localversion }} + kver-extra: ${{ inputs.kver-extra }} + debug-build: ${{ inputs.debug-build }} + qcom-next-pr: ${{ inputs.qcom-next-pr }} + kernel-topics-pr: ${{ inputs.kernel-topics-pr }} + + # --------------------------------------------------------------------------- + # build: generate the Debian source package and submit it to Debusine. + # --------------------------------------------------------------------------- + build: + name: Build (Debusine) + needs: prepare + runs-on: ubuntu-latest + environment: Staging + container: + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + outputs: + workspace: ${{ steps.debusine.outputs.workspace }} + steps: + # Only for the debusine-build action itself: the package being built + # comes from the prepared-source artifact, not from this checkout. + - name: Checkout pkg-linux-qcom + uses: actions/checkout@v4 + + - name: Build in Debusine + id: debusine + uses: ./.github/actions/debusine-build + with: + suite: ${{ inputs.suite }} + flavour: ${{ inputs.flavour }} + debusine-host: ${{ vars.DEBUSINE_HOST }} + debusine-scope: ${{ vars.DEBUSINE_SCOPE }} + debusine-user: ${{ secrets.DEBUSINE_USER }} + debusine-token: ${{ secrets.DEBUSINE_TOKEN }} + debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} + # A staging build: also reads qli-staging, which carries package + # versions that have not been released to qli yet. + extra-build-dep-workspaces: qli qli-staging + + # --------------------------------------------------------------------------- + # publish: download the .deb files from Debusine and upload them to S3. + # Runs on the self-hosted runner, which has direct IAM access. + # --------------------------------------------------------------------------- + publish: + name: Publish to S3 + needs: build + runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + defaults: + run: + shell: bash + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@v5 + with: + repository: qualcomm-linux/debusine-action + ref: main + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + - name: Generate Debusine apt configuration + env: + DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} + DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} + DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + DEBUSINE_WORKSPACE: ${{ needs.build.outputs.workspace }} + SUITE: ${{ inputs.suite }} + run: | + debusine-action/lib/generate-apt-config + + - name: Download .deb packages from Debusine workspace + run: | + set -euxo pipefail + sudo apt-get install -y --no-install-recommends devscripts + mkdir -p "$GITHUB_WORKSPACE/deb-artifacts" + chdist create debusine-workspace + rm -f ~/.chdist/debusine-workspace/etc/apt/sources.list + install -d ~/.chdist/debusine-workspace/etc/apt/sources.list.d + install -d ~/.chdist/debusine-workspace/etc/apt/auth.conf.d + install -m 0644 debusine-ci.sources \ + ~/.chdist/debusine-workspace/etc/apt/sources.list.d/ + install -m 0600 debusine-ci-auth.conf \ + ~/.chdist/debusine-workspace/etc/apt/auth.conf.d/ + chdist apt-get debusine-workspace update + packages=$(chdist apt-cache debusine-workspace search . | awk '{print $1}' | tr '\n' ' ') + [[ -n "$packages" ]] || { echo "ERROR: no packages found in Debusine workspace"; exit 1; } + echo "Packages to download: $packages" + cd "$GITHUB_WORKSPACE/deb-artifacts" + # shellcheck disable=SC2086 + chdist apt-get debusine-workspace download $packages + + - name: Upload .deb packages to S3 + # Keep flavour and suite in the destination so concurrent matrix legs + # cannot overwrite or mix package outputs. Consumers must select the + # flavour and suite they intend to install. + uses: qualcomm-linux/upload-private-artifact-action@aws-v4 + with: + s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} + path: deb-artifacts + destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ diff --git a/.github/workflows/build-kernel-debusine.yml b/.github/workflows/build-kernel-debusine.yml deleted file mode 100644 index d8ae9960..00000000 --- a/.github/workflows/build-kernel-debusine.yml +++ /dev/null @@ -1,248 +0,0 @@ -name: build-debusine - -# Reusable workflow: Debian kernel build via Debusine, then publish. -# -# Called by build-kernel-deb.yml for Debian-family suites (trixie, forky, sid). -# The kernel source is already prepared by the caller's prepare job and shared -# as the kernel-srcpkg artifact. Artifact and workspace identity include both -# flavour and suite so parallel flavours cannot share inputs or outputs. -# -# Publish path is determined by target-workspace: -# target-workspace == '' -> daily path: download .deb files, upload to S3. -# target-workspace != '' -> release path: promote from CI workspace to the -# target Debusine workspace via debusine-action lib/release. -# -# The same input picks the GitHub environment and the workspaces the build -# resolves its build-dependencies from: a Staging run also reads qli-staging, -# a Production run reads qli alone. -# -# Jobs: -# build (debusine-pkg-builder container): generates .dsc, submits to Debusine. -# publish (self-hosted runner): S3 upload (daily path only). -# release (debusine-pkg-builder container): Debusine promotion (release path only). - -on: - workflow_call: - inputs: - suite: - description: 'Target Debian suite (trixie, forky, sid)' - type: string - required: true - flavour: - description: 'Kernel flavour, isolating artifacts, Debusine workspaces and S3 paths' - type: string - required: true - srcpkg: - description: 'Source package name (e.g. linux-qcom-next)' - type: string - default: 'linux-qcom-next' - debusine-parent-workspace: - description: 'Parent Debusine workspace for CI child workspace creation' - type: string - default: 'qli-ci' - target-workspace: - description: 'Debusine target workspace for release promotion (empty = daily/S3 path)' - type: string - default: '' - secrets: - DEBUSINE_USER: - required: true - DEBUSINE_TOKEN: - required: true - DEBUSINE_RELEASE_TOKEN: - required: false - -permissions: - contents: read - packages: read - -env: - DEBUSINE_ACTION_REF: main - -jobs: - # --------------------------------------------------------------------------- - # build: generate Debian source package and submit to Debusine. - # --------------------------------------------------------------------------- - build: - name: Build (Debusine) - runs-on: ubuntu-latest - container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie - options: --user 0:0 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - outputs: - workspace: ${{ steps.build-debusine.outputs.workspace }} - workspace_url: ${{ steps.build-debusine.outputs.workspace_url }} - srcpkg_version: ${{ steps.generate-srcpkg.outputs.srcpkg_version }} - defaults: - run: - shell: bash - environment: ${{ inputs.target-workspace != '' && 'Production' || 'Staging' }} - steps: - - name: Checkout debusine-action helpers - uses: actions/checkout@v5 - with: - repository: qualcomm-linux/debusine-action - ref: ${{ env.DEBUSINE_ACTION_REF }} - path: debusine-action - fetch-depth: 1 - sparse-checkout: | - lib - - - name: Download prepared source tree - uses: actions/download-artifact@v4 - with: - name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} - path: /tmp/srcpkg-artifact - - - name: Extract prepared source tree - run: | - mkdir srcpkg - tar xzf /tmp/srcpkg-artifact/*.tar.gz \ - -C srcpkg --strip-components=1 - - - name: Generate source package - id: generate-srcpkg - env: - SUITE: ${{ inputs.suite }} - DEBUSINE_ASSEMBLE_ORIG: "true" - run: | - SUITE="$SUITE" debusine-action/lib/generate-source-package - - - name: Build in Debusine - id: build-debusine - env: - GITHUB_REPOSITORY_ID: ${{ github.repository_id }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - JOB_INDEX: ${{ inputs.flavour }}-${{ inputs.suite }} - DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} - DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} - DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_PARENT_WORKSPACE: ${{ inputs.debusine-parent-workspace }} - SUITE: ${{ inputs.suite }} - # Staging runs may build against packages that have not been released - # to qli yet, so they resolve build-dependencies from qli-staging too. - EXTRA_BUILD_DEP_WORKSPACES: ${{ inputs.target-workspace != '' && 'qli' || 'qli qli-staging' }} - run: | - # No -x here: DEBUSINE_TOKEN is in env, keep xtrace off. - set -euo pipefail - debusine-action/lib/build - workspace=$(sed -n 's/^workspace=//p' "$GITHUB_OUTPUT") - echo "workspace_url=https://${DEBUSINE_HOST}/${DEBUSINE_SCOPE}/${workspace}/" >> "$GITHUB_OUTPUT" - - - name: Note Debusine workspace URL - env: - WORKSPACE_URL: ${{ steps.build-debusine.outputs.workspace_url }} - run: | - echo "Debusine Workspace URL: $WORKSPACE_URL" >> "$GITHUB_STEP_SUMMARY" - - # --------------------------------------------------------------------------- - # publish: download .deb files from Debusine and upload to S3. - # Daily path only (target-workspace is empty). - # --------------------------------------------------------------------------- - publish: - name: Publish to S3 - needs: build - if: ${{ needs.build.result == 'success' && inputs.target-workspace == '' }} - runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] - defaults: - run: - shell: bash - steps: - - name: Checkout debusine-action helpers - uses: actions/checkout@v5 - with: - repository: qualcomm-linux/debusine-action - ref: ${{ env.DEBUSINE_ACTION_REF }} - path: debusine-action - fetch-depth: 1 - sparse-checkout: | - lib - - - name: Generate Debusine apt configuration - env: - DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} - DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} - DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_WORKSPACE: ${{ needs.build.outputs.workspace }} - SUITE: ${{ inputs.suite }} - run: | - debusine-action/lib/generate-apt-config - - - name: Download .deb packages from Debusine workspace - run: | - set -euxo pipefail - sudo apt-get install -y --no-install-recommends devscripts - mkdir -p "$GITHUB_WORKSPACE/deb-artifacts" - chdist create debusine-workspace - rm -f ~/.chdist/debusine-workspace/etc/apt/sources.list - install -d ~/.chdist/debusine-workspace/etc/apt/sources.list.d - install -d ~/.chdist/debusine-workspace/etc/apt/auth.conf.d - install -m 0644 debusine-ci.sources \ - ~/.chdist/debusine-workspace/etc/apt/sources.list.d/ - install -m 0600 debusine-ci-auth.conf \ - ~/.chdist/debusine-workspace/etc/apt/auth.conf.d/ - chdist apt-get debusine-workspace update - packages=$(chdist apt-cache debusine-workspace search . | awk '{print $1}' | tr '\n' ' ') - [[ -n "$packages" ]] || { echo "ERROR: no packages found in Debusine workspace"; exit 1; } - echo "Packages to download: $packages" - cd "$GITHUB_WORKSPACE/deb-artifacts" - # shellcheck disable=SC2086 - chdist apt-get debusine-workspace download $packages - - - name: Upload .deb packages to S3 - uses: qualcomm-linux/upload-private-artifact-action@aws-v4 - with: - s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} - path: deb-artifacts - destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ - - # --------------------------------------------------------------------------- - # release: promote packages from CI workspace to target Debusine workspace. - # Release path only (target-workspace is non-empty). - # --------------------------------------------------------------------------- - release: - name: Release to Debusine - needs: build - if: ${{ needs.build.result == 'success' && inputs.target-workspace != '' }} - runs-on: ubuntu-latest - environment: Production - container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie - options: --user 0:0 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - defaults: - run: - shell: bash - steps: - - name: Checkout debusine-action helpers - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 - with: - persist-credentials: false - repository: qualcomm-linux/debusine-action - ref: ${{ env.DEBUSINE_ACTION_REF }} - path: debusine-action - fetch-depth: 1 - sparse-checkout: | - lib - - - name: Promote packages to target workspace - env: - DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} - DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} - DEBUSINE_CI_WORKSPACE: ${{ needs.build.outputs.workspace }} - DEBUSINE_TARGET_WORKSPACE: ${{ inputs.target-workspace }} - SRCPKG_NAME: ${{ inputs.srcpkg }} - SRCPKG_VERSION: ${{ needs.build.outputs.srcpkg_version }} - SUITE: ${{ inputs.suite }} - run: | - set -ex - debusine-action/lib/release diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 97a15264..1e4a8aa1 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -1,43 +1,225 @@ name: build-kernel-ubuntu -# Reusable workflow: Ubuntu kernel build via build-kernel.sh. +# Build a kernel package for an Ubuntu-family suite and publish it to S3. # -# Called by build-kernel-deb.yml when the target suite is Ubuntu-family -# (noble, questing, resolute). The kernel source has already been cloned, -# patched, and prepared (debian/ injected, debian/control + debian/changelog -# generated) by the prepare job in the caller workflow. This workflow -# downloads the kernel-srcpkg artifact and runs build-kernel.sh --skip-prepare -# to go directly to the build step. +# Called per matrix entry by daily.yml and pr-build.yml, which select the +# Ubuntu entries with resolve-matrix.py --family ubuntu and call only this +# workflow for them. The package is built in a suite-matched pkg-builder +# container on the self-hosted runner rather than in Debusine, which builds +# only the Debian family. # -# The artifact is shared from the caller via github.run_id. Build and S3 upload -# run on the self-hosted runner, which has direct IAM access. +# Ubuntu has no release path: promotion runs through Debusine, so a Release +# entry for an Ubuntu suite is rejected by resolve-matrix.py rather than built +# here and quietly not promoted. +# +# Jobs: +# prepare (self-hosted): clone, patch, inject debian/, generate the source. +# build (self-hosted): build the .deb files and upload them to S3. on: workflow_call: inputs: - distro: - description: Target Ubuntu suite (noble, questing, resolute) + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + type: string + default: 'qcom-next-resolute' + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + type: string + default: 'qcom-next' + suite: + description: 'Target Ubuntu suite (noble, questing, resolute)' + type: string + default: 'resolute' + kernel-branch: + description: 'Source branch or pinned tag used by branch_tip or pinned_ref' + type: string + default: 'qcom-next' + git-clone: + description: 'Kernel repository URL' + type: string + default: 'https://github.com/qualcomm-linux/kernel' + ref-strategy: + description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' + type: string + default: 'latest_tag' + tag-pattern: + description: 'Tag glob used when ref-strategy=latest_tag' + type: string + default: 'qcom-next-*' + srcpkg: + description: 'Source package name' + type: string + default: 'linux-qcom-next' + binpkg: + description: 'Binary metapackage name' + type: string + default: 'linux-image-qcom-next' + kernel-config: + description: 'Extra config fragments applied on top of all of debian/config-available/' + type: string + default: '' + dkms: + description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' + type: string + default: '' + debian-revision: + description: 'Debian revision component of the package version' + type: string + default: '0qli~' + localversion: + description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' + type: string + default: '' + kver-extra: + description: 'Extra suffix appended to the package version' + type: string + default: '' + debug-build: + description: 'Enable the debug build configuration' + type: boolean + default: false + qcom-next-pr: + description: 'qcom-next PR numbers to merge (e.g. "42 43")' + type: string + default: '' + kernel-topics-pr: + description: 'kernel-topics PR numbers to apply as patches' type: string + default: '' + + # Ad-hoc build of one Ubuntu suite. + workflow_dispatch: + inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' required: true + default: 'qcom-next-resolute' + type: string flavour: - description: 'Kernel flavour, isolating prepared-source artifacts and published packages' + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + required: true + default: 'qcom-next' type: string + suite: + description: 'Target Ubuntu suite (noble, questing, resolute). A Debian suite here is rejected before anything is built; dispatch build-kernel-debian for those.' required: true + default: 'resolute' + type: string + ref-strategy: + description: 'Kernel ref strategy' + required: true + default: 'latest_tag' + type: choice + options: + - latest_tag + - branch_tip + - pinned_ref + kernel-branch: + description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' + required: false + default: 'qcom-next' + tag-pattern: + description: 'Tag glob used only by latest_tag (ignored for branch_tip and pinned_ref)' + required: false + default: 'qcom-next-*' + git-clone: + description: 'Advanced: custom kernel repository URL' + required: false + default: 'https://github.com/qualcomm-linux/kernel' + srcpkg: + description: 'Advanced: Debian source package name' + required: false + default: 'linux-qcom-next' + binpkg: + description: 'Advanced: kernel image metapackage name' + required: false + default: 'linux-image-qcom-next' + kernel-config: + description: 'Advanced: extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' + required: false + default: '' + dkms: + description: 'Advanced: out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' + required: false + default: '' + debian-revision: + description: 'Advanced: Debian revision override (empty takes the Daily revision the matrix gives this build and suite)' + required: false + default: '' + localversion: + description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' + required: false + default: '' + kver-extra: + description: 'Advanced: extra package-version suffix (e.g. -ci42)' + required: false + default: '' + debug-build: + description: 'Advanced: enable the debug build configuration' + type: boolean + required: false + default: false + qcom-next-pr: + description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' + type: string + required: false + default: '' + kernel-topics-pr: + description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' + type: string + required: false + default: '' permissions: contents: read packages: read jobs: + # --------------------------------------------------------------------------- + # prepare: clone kernel source, inject packaging, run prepare-source.sh. + # --------------------------------------------------------------------------- + prepare: + name: Prepare kernel source + runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + steps: + - name: Checkout pkg-linux-qcom + uses: actions/checkout@v4 + + - name: Prepare kernel source + uses: ./.github/actions/prepare-kernel-source + with: + build: ${{ inputs.build }} + suite: ${{ inputs.suite }} + expect-family: ubuntu + flavour: ${{ inputs.flavour }} + kernel-url: ${{ inputs.git-clone }} + ref-strategy: ${{ inputs.ref-strategy }} + kernel-branch: ${{ inputs.kernel-branch }} + tag-pattern: ${{ inputs.tag-pattern }} + srcpkg: ${{ inputs.srcpkg }} + binpkg: ${{ inputs.binpkg }} + kernel-config: ${{ inputs.kernel-config }} + dkms: ${{ inputs.dkms }} + debian-revision: ${{ inputs.debian-revision }} + localversion: ${{ inputs.localversion }} + kver-extra: ${{ inputs.kver-extra }} + debug-build: ${{ inputs.debug-build }} + qcom-next-pr: ${{ inputs.qcom-next-pr }} + kernel-topics-pr: ${{ inputs.kernel-topics-pr }} + + # --------------------------------------------------------------------------- + # build: build the prepared source in a suite-matched container, then publish. + # --------------------------------------------------------------------------- build: name: Build kernel package + needs: prepare runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] env: - DISTRO: ${{ inputs.distro }} + DISTRO: ${{ inputs.suite }} defaults: run: shell: bash - steps: - name: Checkout pkg-linux-qcom uses: actions/checkout@v4 @@ -60,7 +242,7 @@ jobs: - name: Download prepared source tree uses: actions/download-artifact@v4 with: - name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.distro }} + name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} path: /tmp/srcpkg-artifact - name: Extract prepared source tree @@ -74,12 +256,12 @@ jobs: - name: Build kernel package run: | - # Kernel source was cloned, patched, and prepared by the caller's - # prepare job. Pass --local-source and --skip-prepare so - # build-kernel.sh goes directly to the build step. - # debian/control, debian/changelog, and all config fragments are - # already baked into the extracted source tree; no re-derivation - # of LOCALVERSION, kver-extra, or debug config is needed here. + # Kernel source was cloned, patched, and prepared by the prepare job. + # Pass --local-source and --skip-prepare so build-kernel.sh goes + # directly to the build step. debian/control, debian/changelog, and + # all config fragments are already baked into the extracted source + # tree; no re-derivation of LOCALVERSION, kver-extra, or debug config + # is needed here. ./build-kernel.sh \ --build-mode docker \ --distro "$DISTRO" \ @@ -89,11 +271,11 @@ jobs: - name: Upload .deb packages to S3 # build-kernel.sh outputs to kernel-build// by default. - # Keep kernel variant and suite in the destination so concurrent matrix - # legs cannot overwrite or mix package outputs. Consumers must select - # the variant and suite they intend to install. + # Keep flavour and suite in the destination so concurrent matrix legs + # cannot overwrite or mix package outputs. Consumers must select the + # flavour and suite they intend to install. uses: qualcomm-linux/upload-private-artifact-action@aws-v4 with: s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} - path: kernel-build/${{ inputs.distro }} - destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.distro }}/${{ github.run_id }}-${{ github.run_attempt }}/ + path: kernel-build/${{ inputs.suite }} + destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 06abc389..f78b97f3 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -9,9 +9,9 @@ name: daily # One entry is one generated package, so ci/scripts/resolve-matrix.py selects # the entries with type: Daily and passes them through as they stand. Every # field (name, suite, flavour, srcpkg, binpkg, kernel_config, dkms, -# debian_revision, git_clone, branch_or_tag, ref_strategy) becomes a -# build-kernel-deb.yml input. name labels the job; flavour is what reaches -# the built kernel's version and its published paths. +# debian_revision, git_clone, branch_or_tag, ref_strategy) becomes an input +# of the build workflow for the entry's family. name labels the job; flavour +# is what reaches the built kernel's version and its published paths. # # To add a daily build target: add an entry to ci/build-matrix.yaml with # type: Daily. No workflow changes needed. @@ -59,7 +59,8 @@ jobs: name: Configure build matrix runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + debian: ${{ steps.set-matrix.outputs.debian }} + ubuntu: ${{ steps.set-matrix.outputs.ubuntu }} steps: - uses: actions/checkout@v4 @@ -83,7 +84,7 @@ jobs: echo "Source: ci/build-matrix.yaml (full Daily matrix)" args=(--type Daily) ;; - "Selected variants") + "Selected builds") echo "Source: ci/build-matrix.yaml (builds: $INPUT_BUILDS)" args=(--type Daily --build "$INPUT_BUILDS") ;; @@ -96,27 +97,45 @@ jobs: exit 1 ;; esac - MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}") - echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" - echo "Matrix entries: $(echo "$MATRIX" | jq length)" + # One selection, split by build path. The two families are built by + # different workflows, so each gets the entries it builds and neither + # run starts a job for the other. A narrowed dispatch can leave one + # side empty, which --allow-empty reports as [] rather than as a + # failure; a name that matches nothing still fails, in both calls. + for family in debian ubuntu; do + MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}" \ + --family "$family" --allow-empty) + echo "$family=$MATRIX" >> "$GITHUB_OUTPUT" + echo "$family entries: $(echo "$MATRIX" | jq length)" + done # --------------------------------------------------------------------------- - # build: one parallel job per matrix entry. + # build-debian / build-ubuntu: one parallel job per matrix entry. # - # Each entry calls build-kernel-deb.yml as a reusable workflow. The full - # set of matrix fields (suite, srcpkg, binpkg, kernel_config, dkms, etc.) is - # threaded through as inputs. fail-fast is disabled so a failed trixie - # build does not cancel the forky build. + # Two jobs rather than one because the families are built by different + # workflows, on different runners, and published from different places. + # Splitting the matrix by family means each entry starts exactly the jobs + # that build it: a run shows no skipped job for the path it did not take. + # + # Both are named ${{ matrix.name }}, so the checks list reads the same as + # one job would -- the split is in what each entry calls, not in what it is + # called. fail-fast is disabled so a failed trixie build does not cancel the + # forky build. + # + # The if: guards only matter for a narrowed dispatch that selects nothing on + # one path; an empty matrix is an error rather than an empty job list. The + # scheduled full-matrix run has entries on both. # --------------------------------------------------------------------------- - build: + build-debian: name: ${{ matrix.name }} needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.debian != '[]' }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} - uses: ./.github/workflows/build-kernel-deb.yml + include: ${{ fromJson(needs.configure-matrix.outputs.debian) }} + uses: ./.github/workflows/build-kernel-debian.yml with: suite: ${{ matrix.suite }} build: ${{ matrix.name }} @@ -136,4 +155,28 @@ jobs: secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} + + build-ubuntu: + name: ${{ matrix.name }} + needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.ubuntu != '[]' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.configure-matrix.outputs.ubuntu) }} + uses: ./.github/workflows/build-kernel-ubuntu.yml + with: + suite: ${{ matrix.suite }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} + git-clone: ${{ matrix.git_clone }} + kernel-branch: ${{ matrix.branch_or_tag }} + ref-strategy: ${{ matrix.ref_strategy }} + tag-pattern: ${{ matrix.tag_pattern || '' }} + srcpkg: ${{ matrix.srcpkg }} + binpkg: ${{ matrix.binpkg }} + kernel-config: ${{ matrix.kernel_config }} + dkms: ${{ matrix.dkms }} + debian-revision: ${{ matrix.debian_revision }} + localversion: ${{ matrix.localversion || '' }} + kver-extra: ${{ matrix.kver_extra || '' }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index df03c318..7b3f699a 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -30,7 +30,8 @@ jobs: name: Configure build matrix runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + debian: ${{ steps.set-matrix.outputs.debian }} + ubuntu: ${{ steps.set-matrix.outputs.ubuntu }} steps: - uses: actions/checkout@v4 with: @@ -41,24 +42,39 @@ jobs: id: set-matrix run: | set -euo pipefail - MATRIX=$(ci/scripts/resolve-matrix.py --type Daily) - echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" - echo "Matrix entries: $(echo "$MATRIX" | jq length)" + # Split by build path: the families are built by different workflows, + # so a PR gets one job per leg it actually builds and no skipped job + # for the path that leg does not take. + for family in debian ubuntu; do + MATRIX=$(ci/scripts/resolve-matrix.py --type Daily \ + --family "$family" --allow-empty) + echo "$family=$MATRIX" >> "$GITHUB_OUTPUT" + echo "$family entries: $(echo "$MATRIX" | jq length)" + done # --------------------------------------------------------------------------- - # build: one parallel job per matrix entry, mirroring daily.yml. + # build-debian / build-ubuntu: one parallel job per matrix entry, mirroring + # daily.yml. + # + # A PR gets one check per build leg and nothing else: neither workflow has a + # branch to skip, and a PR builds the whole Daily matrix, so both families + # have entries and neither guard fires. # # fail-fast is disabled so a broken trixie leg still leaves forky and # resolute results on the PR. + # + # Neither takes DEBUSINE_RELEASE_TOKEN: a PR build never promotes, so it + # calls the S3-publishing workflow and not release-kernel-debian.yml. # --------------------------------------------------------------------------- - build: + build-debian: name: ${{ matrix.name }} needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.debian != '[]' }} strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} - uses: ./.github/workflows/build-kernel-deb.yml + include: ${{ fromJson(needs.configure-matrix.outputs.debian) }} + uses: ./.github/workflows/build-kernel-debian.yml with: suite: ${{ matrix.suite }} build: ${{ matrix.name }} @@ -77,7 +93,30 @@ jobs: # in the shared Debusine workspace. kver-extra: -pr${{ github.event.number }} secrets: - # No DEBUSINE_RELEASE_TOKEN: PR builds never promote to a release - # workspace, so they leave target-workspace empty. DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + + build-ubuntu: + name: ${{ matrix.name }} + needs: configure-matrix + if: ${{ needs.configure-matrix.outputs.ubuntu != '[]' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.configure-matrix.outputs.ubuntu) }} + uses: ./.github/workflows/build-kernel-ubuntu.yml + with: + suite: ${{ matrix.suite }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} + git-clone: ${{ matrix.git_clone }} + kernel-branch: ${{ matrix.branch_or_tag }} + ref-strategy: ${{ matrix.ref_strategy }} + tag-pattern: ${{ matrix.tag_pattern || '' }} + srcpkg: ${{ matrix.srcpkg }} + binpkg: ${{ matrix.binpkg }} + kernel-config: ${{ matrix.kernel_config }} + dkms: ${{ matrix.dkms }} + debian-revision: ${{ matrix.debian_revision }} + localversion: ${{ matrix.localversion || '' }} + kver-extra: -pr${{ github.event.number }} diff --git a/.github/workflows/release-kernel-debian.yml b/.github/workflows/release-kernel-debian.yml new file mode 100644 index 00000000..759d054b --- /dev/null +++ b/.github/workflows/release-kernel-debian.yml @@ -0,0 +1,210 @@ +name: release-kernel-debian + +# Build a kernel package for a Debian-family suite and promote it to a release +# workspace in Debusine. +# +# Called per matrix entry by release.yml. Identical to build-kernel-debian.yml +# up to the Debusine build -- both share the prepare-kernel-source and +# debusine-build actions -- and differs only in the tail: this one promotes the +# built package to target-workspace rather than publishing it to the daily S3 +# path. Which tail a run wants is decided by the workflow that calls it, so +# neither file carries the other's tail as a job to be skipped. +# +# Only the Debian family has a release path: promotion runs through Debusine, +# and resolve-matrix.py rejects a Release entry for any other suite. +# +# Jobs: +# prepare (self-hosted): clone, patch, inject debian/, generate the source. +# build (debusine-pkg-builder container): submit to Debusine. +# release (debusine-pkg-builder container): promote to the target workspace. + +on: + workflow_call: + inputs: + build: + description: 'Name of this build, as listed in ci/build-matrix.yaml' + type: string + default: 'qcom-next-trixie' + flavour: + description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' + type: string + default: 'qcom-next' + suite: + description: 'Target Debian suite (trixie, forky, sid)' + type: string + default: 'trixie' + kernel-branch: + description: 'Immutable ref to build, from the entry branch_or_tag' + type: string + default: 'qcom-next' + git-clone: + description: 'Kernel repository URL' + type: string + default: 'https://github.com/qualcomm-linux/kernel' + ref-strategy: + description: 'Kernel ref strategy: a Release entry always pins one' + type: string + default: 'pinned_ref' + tag-pattern: + description: 'Tag glob, unused by pinned_ref and carried for symmetry with the Daily inputs' + type: string + default: '' + srcpkg: + description: 'Source package name' + type: string + default: 'linux-qcom-next' + binpkg: + description: 'Binary metapackage name' + type: string + default: 'linux-image-qcom-next' + kernel-config: + description: 'Extra config fragments applied on top of all of debian/config-available/' + type: string + default: '' + dkms: + description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' + type: string + default: '' + debian-revision: + description: 'Debian revision component of the package version' + type: string + default: '0qli' + localversion: + description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' + type: string + default: '' + kver-extra: + description: 'Extra suffix appended to the package version' + type: string + default: '' + debusine-parent-workspace: + description: 'Parent Debusine workspace for CI child workspace creation' + type: string + default: '' + target-workspace: + description: 'Debusine workspace to promote the built package into' + type: string + required: true + secrets: + DEBUSINE_USER: + description: 'Debusine account used to submit release builds' + required: false + DEBUSINE_TOKEN: + description: 'Debusine token for the CI child workspace' + required: false + DEBUSINE_RELEASE_TOKEN: + description: 'Debusine token for promotion to the release target workspace' + required: false + +permissions: + contents: read + packages: read + +jobs: + # --------------------------------------------------------------------------- + # prepare: clone kernel source, inject packaging, run prepare-source.sh. + # --------------------------------------------------------------------------- + prepare: + name: Prepare kernel source + runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + steps: + - name: Checkout pkg-linux-qcom + uses: actions/checkout@v4 + + - name: Prepare kernel source + uses: ./.github/actions/prepare-kernel-source + with: + build: ${{ inputs.build }} + suite: ${{ inputs.suite }} + expect-family: debian + flavour: ${{ inputs.flavour }} + kernel-url: ${{ inputs.git-clone }} + ref-strategy: ${{ inputs.ref-strategy }} + kernel-branch: ${{ inputs.kernel-branch }} + tag-pattern: ${{ inputs.tag-pattern }} + srcpkg: ${{ inputs.srcpkg }} + binpkg: ${{ inputs.binpkg }} + kernel-config: ${{ inputs.kernel-config }} + dkms: ${{ inputs.dkms }} + debian-revision: ${{ inputs.debian-revision }} + localversion: ${{ inputs.localversion }} + kver-extra: ${{ inputs.kver-extra }} + + # --------------------------------------------------------------------------- + # build: generate the Debian source package and submit it to Debusine. + # --------------------------------------------------------------------------- + build: + name: Build (Debusine) + needs: prepare + runs-on: ubuntu-latest + environment: Production + container: + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + outputs: + workspace: ${{ steps.debusine.outputs.workspace }} + srcpkg_version: ${{ steps.debusine.outputs.srcpkg_version }} + steps: + - name: Checkout pkg-linux-qcom + uses: actions/checkout@v4 + + - name: Build in Debusine + id: debusine + uses: ./.github/actions/debusine-build + with: + suite: ${{ inputs.suite }} + flavour: ${{ inputs.flavour }} + debusine-host: ${{ vars.DEBUSINE_HOST }} + debusine-scope: ${{ vars.DEBUSINE_SCOPE }} + debusine-user: ${{ secrets.DEBUSINE_USER }} + debusine-token: ${{ secrets.DEBUSINE_TOKEN }} + debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} + # A release build reads qli alone, so a released kernel is never + # built against something that has not been released alongside it. + extra-build-dep-workspaces: qli + + # --------------------------------------------------------------------------- + # release: promote the built package from the CI workspace to the target. + # --------------------------------------------------------------------------- + release: + name: Release to Debusine + needs: build + runs-on: ubuntu-latest + environment: Production + container: + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: main + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + - name: Promote packages to target workspace + env: + DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} + DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} + DEBUSINE_CI_WORKSPACE: ${{ needs.build.outputs.workspace }} + DEBUSINE_TARGET_WORKSPACE: ${{ inputs.target-workspace }} + SRCPKG_NAME: ${{ inputs.srcpkg }} + SRCPKG_VERSION: ${{ needs.build.outputs.srcpkg_version }} + SUITE: ${{ inputs.suite }} + run: | + set -ex + debusine-action/lib/release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e2e55206..bdbbb1e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,7 +71,7 @@ jobs: echo "Source: ci/build-matrix.yaml (flavour: $INPUT_FLAVOUR, all suites)" args=(--type Release --flavour "$INPUT_FLAVOUR") ;; - "Selected variants") + "Selected builds") echo "Source: ci/build-matrix.yaml (builds: $INPUT_BUILDS)" args=(--type Release --build "$INPUT_BUILDS") ;; @@ -89,8 +89,14 @@ jobs: # build: one parallel job per Release matrix entry. # # Uses the ref strategy and branch_or_tag selected by the matrix. - # Passes target_workspace so build-kernel-debusine.yml triggers the Debusine - # release path (lib/release) instead of the S3 publish path. + # release-kernel-debian.yml builds exactly as the daily workflow does and + # then promotes to target_workspace, rather than publishing to the daily S3 + # path. Choosing the workflow here is what makes that the whole of the run: + # no job in it exists to publish the other way and be skipped. + # + # There is no Ubuntu counterpart. Promotion runs through Debusine, which + # builds only the Debian family, so resolve-matrix.py rejects a Release entry + # for any other suite and this matrix is Debian by construction. # --------------------------------------------------------------------------- build: name: ${{ matrix.name }} @@ -99,7 +105,7 @@ jobs: fail-fast: false matrix: include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} - uses: ./.github/workflows/build-kernel-deb.yml + uses: ./.github/workflows/release-kernel-debian.yml with: suite: ${{ matrix.suite }} build: ${{ matrix.name }} diff --git a/README.md b/README.md index eca38a6d..3fefeeb2 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,24 @@ variant is a matrix change, not a workflow redesign. | --- | --- | --- | | `daily.yml` | Resolves and runs the Daily matrix. | Scheduled daily at `23:00 UTC`, or manual dispatch. | | `release.yml` | Resolves and runs the Release matrix. | Manual dispatch only. | -| `build-kernel-deb.yml` | Reusable orchestrator for one kernel variant and suite. | Manual dispatch or called by Daily and Release. | -| `build-kernel-debusine.yml` | Builds Debian suites in Debusine and either publishes Daily artifacts or promotes Releases. | Called by `build-kernel-deb.yml`. | -| `build-kernel-ubuntu.yml` | Builds Ubuntu-family suites with the Docker path. | Called by `build-kernel-deb.yml`. | +| `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine and publishes it to S3. | Manual dispatch or called by Daily and PR build. | +| `build-kernel-ubuntu.yml` | Builds one Ubuntu-suite entry on the Docker path and publishes it to S3. | Manual dispatch or called by Daily and PR build. | +| `release-kernel-debian.yml` | Builds one Debian-suite entry in Debusine and promotes it to the release workspace. | Called by Release. | + +The three build workflows share their steps through two composite actions +rather than through a common orchestrator workflow: + +| Action | Used by | +| --- | --- | +| `.github/actions/prepare-kernel-source` | All three, as the `prepare` job. | +| `.github/actions/debusine-build` | The two Debian workflows, as the `build` job. | + +Which of them a build leg calls is decided by the caller, from the entry's +suite: `resolve-matrix.py --family debian|ubuntu` splits the selection, and +each family's entries call only the workflow that builds them. Nothing inside a +build workflow is conditional on the suite or on whether the run releases, so a +run starts exactly the jobs it needs and shows no skipped job for a path it did +not take. ### Daily @@ -171,9 +186,15 @@ Release is the controlled promotion path. the release credential and enforces the required approval gate before promotion to `qli`. -Direct `build-kernel-deb.yml` dispatches are build-only. Release promotion is +Direct `build-kernel-debian.yml` and `build-kernel-ubuntu.yml` dispatches are +build-only: neither has a promotion path to offer. Release promotion is initiated exclusively by `release.yml`, which owns the target workspace and -production release controls. +production release controls, and is the only caller of +`release-kernel-debian.yml`. + +Only the Debian family has a release path at all, because promotion runs +through Debusine. `resolve-matrix.py` rejects a `Release` entry for any other +suite rather than letting it build and then silently not promote. ## Matrix Model @@ -236,10 +257,11 @@ face value: - Where a suite has both, its `Daily` revision is its `Release` revision plus a trailing `~`. -`build-kernel-deb.yml`'s direct-dispatch path has no matrix context of its own, -so when its `debian-revision` input is empty it looks up the `Daily` entry for -the variant and suite it was given (`resolve-matrix.py --field -debian_revision`) and builds at the revision the daily build would have used. +A direct dispatch of a build workflow has no matrix context of its own, so when +its `debian-revision` input is empty the `prepare-kernel-source` action looks up +the `Daily` entry for the build name and suite it was given (`resolve-matrix.py +--field debian_revision`) and builds at the revision the daily build would have +used. Each entry has a distinct prepared-source artifact, Debusine child workspace, and S3 path keyed by `flavour + suite`. This prevents two flavours that both @@ -277,16 +299,22 @@ This document covers the CI generator. For the packaging internals: `debian/rule targets, the config fragment merge pipeline, DKMS module bundling and the produced package layout see [debian/README.md](debian/README.md). +Both branches below are taken in the caller, when the matrix is resolved: the +family from the entry's suite, the tail from which workflow is running. By the +time a build workflow starts, there is nothing left to decide. + ```mermaid flowchart LR - IN["Matrix variant + suite input"] --> R{"Resolve suite family"} + IN["Matrix entries"] --> R{"resolve-matrix.py\n--family"} - R -->|"trixie · forky"| DEB["Debian path\nbuild-kernel-debusine.yml\nGenerate source package\nSubmit with lib/build\nDebusine builds binaries"] - R -->|"resolute"| UBU["Ubuntu path\nbuild-kernel-ubuntu.yml\nbuild-kernel.sh in Docker\nBuild binary packages"] + R -->|"debian: trixie · forky"| DT{"Which caller"} + R -->|"ubuntu: resolute"| UBU["build-kernel-ubuntu.yml\nbuild-kernel.sh in Docker\nBuild binary packages"] - DEB --> DOUT{"Build type"} - DOUT -->|Daily| S3["Download .deb files\nPublish to S3"] - DOUT -->|Release| QLI["Promote source and binaries\nto qli"] + DT -->|"daily.yml · pr-build.yml"| DEB["build-kernel-debian.yml\nGenerate source package\nSubmit with lib/build\nDebusine builds binaries"] + DT -->|"release.yml"| REL["release-kernel-debian.yml\nSame build, release tail"] + + DEB --> S3["Download .deb files\nPublish to S3"] + REL --> QLI["Promote source and binaries\nto qli"] UBU --> US3["Publish .deb files to S3"] ``` @@ -298,23 +326,26 @@ flowchart LR flowchart TD subgraph triggers[Triggers] A1["daily.yml\nScheduled full matrix"] - A2["daily.yml\nManual full or filtered variant + suite"] - A3["release.yml\nManual full or filtered variant + suite"] - A4["build-kernel-deb.yml\nManual one-off build"] + A2["daily.yml\nManual full or filtered builds"] + A5["pr-build.yml\nFull Daily matrix on every PR"] + A3["release.yml\nManual full or filtered builds"] + A4["build-kernel-debian.yml\nbuild-kernel-ubuntu.yml\nManual one-off build"] end subgraph matrix[Matrix entry points] - B1["Daily configure-matrix\nFlatten Daily rows"] - B2["Daily variant + suite legs\nqcom-next / trixie · forky · resolute\nqcom-next-debug / trixie · forky"] - B3["Release configure-matrix\nFlatten Release rows"] - B4["Release variant + suite legs\nqcom-next / trixie · forky\nqcom-next-debug / trixie · forky"] + B1["configure-matrix\nDaily entries, split by family"] + B2["build-debian legs\nqcom-next · qcom-next-debug · qcom-arduino\nmainline · next / trixie · forky"] + B5["build-ubuntu legs\nqcom-next / resolute"] + B3["configure-matrix\nRelease entries, Debian by construction"] + B4["build legs\nqcom-next · qcom-next-debug / trixie · forky"] end - subgraph orchestrator[build-kernel-deb.yml] - C1["resolve\nClassify suite family"] - C2["prepare\nClone selected kernel ref\nRun prepare-source.sh\nUpload kernel-srcpkg-variant-suite"] - C3["debusine-build\nDebian suites only"] - C4["ubuntu-build\nUbuntu suites only"] + subgraph build[One build workflow per leg] + C2["prepare\nprepare-kernel-source action\nClone ref, run prepare-source.sh\nUpload kernel-srcpkg-flavour-suite"] + C3["build\ndebusine-build action"] + C4["build\nbuild-kernel.sh in Docker"] + C5["publish\nDownload .deb files, upload to S3"] + C6["release\nPromote to target workspace"] end subgraph outputs[Outputs] @@ -324,16 +355,25 @@ flowchart TD A1 --> B1 A2 --> B1 + A5 --> B1 A3 --> B3 - B1 --> B2 --> C1 - B3 --> B4 --> C1 - A4 --> C1 - C1 --> C2 + B1 --> B2 & B5 + B3 --> B4 + B2 --> C2 + B5 --> C2 + B4 --> C2 + A4 --> C2 C2 --> C3 & C4 - C3 --> D1 & D2 + C3 --> C5 & C6 C4 --> D1 + C5 --> D1 + C6 --> D2 ``` +Every leg runs every job drawn under it. `build-debian` and `build-ubuntu` legs +reach different build workflows, and `publish` and `release` belong to +different ones, so no leg starts a job it will skip. + ### Prepare stage ```mermaid @@ -452,15 +492,18 @@ out-of-tree module builds are required. ## Manual Builds -Use **Actions** → **build-kernel-deb** for a one-off build. It is an explicit -override workflow, not a matrix-derived delivery flow: use `daily.yml` and -`release.yml` for normal Daily and Release operations. +Use **Actions** → **build-kernel-debian** or **build-kernel-ubuntu** for a +one-off build, picking the one that builds the suite you want. These are +explicit override workflows, not matrix-derived delivery flows: use `daily.yml` +and `release.yml` for normal Daily and Release operations. Neither promotes. `build`, `suite`, and `ref-strategy` are the required build selection. All remaining package, configuration, and PR inputs are advanced overrides for -validation or debugging. Variant and suite are free-text matrix values rather -than static dropdowns, so adding a matrix entry never requires editing the -workflow UI. +validation or debugging. Build name and suite are free-text matrix values +rather than static dropdowns, so adding a matrix entry never requires editing +the workflow UI. A suite belonging to the other family is rejected by the +`prepare` job before anything is cloned or built, so the only cost of picking +the wrong workflow is a fast failure. The available inputs are: diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index 3f5082ec..8151cfdf 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -18,6 +18,7 @@ ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie,qcom-next-forky ci/scripts/resolve-matrix.py --type Release --flavour qcom-next + ci/scripts/resolve-matrix.py --type Daily --family ubuntu --allow-empty ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie \\ --field debian_revision @@ -31,6 +32,14 @@ flavour spans its suites, so this asks for one kernel everywhere it is built. --suite SUITES Select only these suites, comma-separated. + --family FAMILY Select only the builds taking one build path, + debian or ubuntu. Derived from suite, so it + selects by how a build is built rather than by + naming every suite that is built that way. + --allow-empty Print [] rather than failing when the filters + select nothing. For a caller asking each family + for the same selection, where one of them + having nothing to build is an ordinary answer. --field NAME Print just this field of the single selected entry, unquoted. Errors unless exactly one entry matches. @@ -39,12 +48,14 @@ Filters combine: an entry must match every filter given. Every name in a filter must match at least one entry of the selected type, so a typo or a - stale name fails instead of quietly narrowing the build set. + stale name fails instead of quietly narrowing the build set. That check is + per filter, so --allow-empty still rejects a name that matches nothing. Output: Without --field, a compact JSON array of the selected entries, ready for a - GitHub Actions matrix `include`. kernel_config and dkms are joined into the - comma-separated strings that build-kernel-deb.yml's kernel-config and dkms + GitHub Actions matrix `include`. Each entry carries a derived family field + naming its build path. kernel_config and dkms are joined into the + comma-separated strings that the build workflows' kernel-config and dkms inputs, and prepare-source.sh's --kernel-config and --dkms, expect; every other field is passed through as written. @@ -76,6 +87,20 @@ DELIVERY_TYPES = ("Daily", "Release") REF_STRATEGIES = ("latest_tag", "branch_tip", "pinned_ref") +# Which build path a suite takes. Debian-family suites are built by Debusine; +# everything else is built in a suite-matched docker container on the +# self-hosted runner. The two paths are different workflows with different +# runners, containers and publish steps, so a caller selects entries by family +# and calls the one workflow that builds them -- rather than every build leg +# starting both and skipping one. +DEBIAN_SUITES = ("trixie", "forky", "sid", "unstable", "bookworm") +FAMILIES = ("debian", "ubuntu") + + +def family_for(suite): + """Return the build family a suite belongs to.""" + return "debian" if suite in DEBIAN_SUITES else "ubuntu" + # A delivery type constrains how its kernel ref is chosen: a Daily build tracks # something moving, a Release build is pinned to an immutable ref. REF_STRATEGIES_FOR_TYPE = { @@ -272,6 +297,17 @@ def check_entry(entry, report): if delivery_type == "Release": if not entry.get("target_workspace"): report("missing or invalid target_workspace") + # Promotion runs through Debusine, and only the Debian family is built + # there. An Ubuntu Release entry would build the package and then have + # no workspace to promote it from, publishing to the daily S3 path and + # reporting success without ever releasing anything. + suite = entry.get("suite") + if isinstance(suite, str) and family_for(suite) != "debian": + report( + f"Release entries must target a Debian suite, not {suite}; " + "promotion runs through Debusine, which builds " + + ", ".join(DEBIAN_SUITES) + ) elif "target_workspace" in entry: report("target_workspace is only valid for Release") @@ -489,6 +525,12 @@ def load_matrix(path): + "\n".join(f" - {error}" for error in errors) ) + # Derived, never written: family follows from suite, so the matrix cannot + # state one that disagrees with the suite it is built for. Attached after + # validation, which rejects family as an unknown field on an entry. + for entry in builds: + entry["family"] = family_for(entry["suite"]) + return builds @@ -517,10 +559,16 @@ def select(builds, delivery_type, filters): def unmatched_filters(builds, delivery_type, filters): - """Names asked for that no entry of this delivery type offers.""" + """Names asked for that no entry of this delivery type offers. + + family is exempt: it routes a selection to a build path rather than naming + something in the matrix, argparse already restricts it to a real family, + and a type having no builds on one path is an ordinary state of the matrix + rather than a mistake in the request. + """ missing = [] for field, wanted in filters.items(): - if wanted is None: + if wanted is None or field == "family": continue available = { entry[field] for entry in builds if entry["type"] == delivery_type @@ -573,6 +621,17 @@ def main(): parser.add_argument( "--suite", default="", help="select only these suites, comma-separated" ) + parser.add_argument( + "--family", + default="", + choices=("",) + FAMILIES, + help="select only the builds taking this build path", + ) + parser.add_argument( + "--allow-empty", + action="store_true", + help="print [] instead of failing when the filters select nothing", + ) parser.add_argument( "--field", default="", @@ -588,6 +647,7 @@ def main(): "name": parse_filter(args.build), "flavour": parse_filter(args.flavour), "suite": parse_filter(args.suite), + "family": parse_filter(args.family), } what = describe_selection(args.type, filters) @@ -603,6 +663,13 @@ def main(): selected = select(builds, args.type, filters) if not selected: + # A caller splitting one dispatch across both build paths asks each + # family for the same selection, and one of them legitimately has + # nothing to build. Every name in the request still had to match + # something above, so this is an empty intersection, not a typo. + if args.allow_empty and not args.field: + print("[]") + return sys.exit(f"ERROR: No matrix entries found for {what}") if not args.field: From d963bcb0c5a1e6895d7869e063c402ccc3c89867 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 22/71] ci: pass prepare's values as step outputs and drop checkout credentials zizmor raised two findings against the new action and workflows. prepare-kernel-source wrote its intermediate values to GITHUB_ENV, which puts them in the environment of every later step in the job rather than only the steps that want them, and lets a value containing a newline define environment variables of its own -- LD_PRELOAD or NODE_OPTIONS being the interesting ones. Not all of these values are ours to trust: the kernel ref is whatever the remote's tag list offers, and on pr-build.yml the inputs come from the build matrix as the pull request wrote it, since a PR is built against its own ci/build-matrix.yaml. So the steps now hand their values on as step outputs, and each consumer names in its own env: block what it reads. That is the same data flowing the same way, but scoped to the steps that asked for it. The resolved kernel ref is also rejected outright if it contains a newline, because that one is chosen furthest from us and step outputs are injectable in the same manner if less usefully. The checkouts leave the GitHub token in .git/config, which the prepare job then packs a sibling directory into an artifact next to. None of these checkouts is fetched from or pushed to afterwards: the kernel is cloned separately with its own remote, and the PR-merge steps fetch inside that clone rather than this one. So they all set persist-credentials: false, as release-kernel-debian.yml's debusine-action checkout and pr-build.yml and release.yml already did. daily.yml's configure-matrix checkout gets the same treatment. zizmor did not flag it, having only reviewed the diff, but it is the same checkout doing the same job as the two that were already set that way. No build changes what it produces, and the job graph is as it was: 32 jobs across 11 legs for a daily or PR run, 12 across 4 for a release, none skipped. Signed-off-by: Christopher Obbard --- .github/actions/debusine-build/action.yml | 1 + .../actions/prepare-kernel-source/action.yml | 116 +++++++++++++++--- .github/workflows/build-kernel-debian.yml | 5 + .github/workflows/build-kernel-ubuntu.yml | 5 + .github/workflows/daily.yml | 2 + .github/workflows/release-kernel-debian.yml | 4 + 6 files changed, 117 insertions(+), 16 deletions(-) diff --git a/.github/actions/debusine-build/action.yml b/.github/actions/debusine-build/action.yml index 0eff3ade..6355aea6 100644 --- a/.github/actions/debusine-build/action.yml +++ b/.github/actions/debusine-build/action.yml @@ -66,6 +66,7 @@ runs: - name: Checkout debusine-action helpers uses: actions/checkout@v5 with: + persist-credentials: false repository: qualcomm-linux/debusine-action ref: ${{ inputs.debusine-action-ref }} path: debusine-action diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index ae171847..bfe94518 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -11,8 +11,14 @@ description: > # The caller must have checked out pkg-linux-qcom into the workspace first: # debian/ and ci/ come from that checkout, and so does this action. # -# Runs on the self-hosted arm64 runner. Leaves KERNEL_DIR, KERNEL_REF, -# KERNEL_SHA, LOCALVERSION and DEBIAN_REVISION in the job environment. +# Runs on the self-hosted arm64 runner, and leaves nothing behind in the job +# environment: values pass between these steps as step outputs, each consumer +# naming what it reads. Writing them to GITHUB_ENV instead would put them in +# the environment of every later step in the job, including steps outside this +# action, and a newline in one would let the value that carried it define +# environment variables of its own. Several of these values are not ours: the +# kernel ref comes from a remote's tag list, and on pr-build.yml the inputs +# come from the build matrix as the pull request wrote it. inputs: build: @@ -77,6 +83,7 @@ runs: using: composite steps: - name: Resolve build environment + id: env shell: bash env: SUITE_INPUT: ${{ inputs.suite }} @@ -103,24 +110,28 @@ runs: exit 1 fi - echo "DISTRO=$SUITE_INPUT" >> "$GITHUB_ENV" - echo "DOCKER_DISTRO=$DOCKER_DISTRO" >> "$GITHUB_ENV" + echo "distro=$SUITE_INPUT" >> "$GITHUB_OUTPUT" + echo "docker_distro=$DOCKER_DISTRO" >> "$GITHUB_OUTPUT" echo "Suite $SUITE_INPUT is $FAMILY-family and builds in the $DOCKER_DISTRO image" - name: Checkout docker-pkg-build uses: actions/checkout@v4 with: + persist-credentials: false repository: qualcomm-linux/docker-pkg-build ref: main path: docker-pkg-build - name: Build docker image shell: bash + env: + DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} run: | ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DOCKER_DISTRO" docker image ls - name: Resolve kernel ref + id: ref shell: bash env: KERNEL_URL_INPUT: ${{ inputs.kernel-url }} @@ -134,8 +145,6 @@ runs: echo "::error::build name must use lowercase letters, digits, and internal hyphens" exit 1 } - echo "KERNEL_URL=$KERNEL_URL_INPUT" >> "$GITHUB_ENV" - case "$REF_STRATEGY_INPUT" in latest_tag) KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh \ @@ -152,41 +161,91 @@ runs: ;; esac - echo "KERNEL_REF=$KERNEL_REF" >> "$GITHUB_ENV" + # A ref carrying a newline would otherwise define outputs of its own. + # The remote chooses this value, so check it before writing it out. + [[ "$KERNEL_REF" == *$'\n'* ]] && { + echo "::error::resolved kernel ref contains a newline" + exit 1 + } + + echo "kernel_ref=$KERNEL_REF" >> "$GITHUB_OUTPUT" echo "Build: $BUILD_INPUT" echo "Ref strategy: $REF_STRATEGY_INPUT" echo "Resolved kernel ref: $KERNEL_REF" - name: Clone kernel source + id: clone shell: bash + env: + KERNEL_URL: ${{ inputs.kernel-url }} + KERNEL_REF: ${{ steps.ref.outputs.kernel_ref }} run: | set -euo pipefail KERNEL_DIR="$GITHUB_WORKSPACE/kernel-source" - echo "KERNEL_DIR=$KERNEL_DIR" >> "$GITHUB_ENV" + echo "kernel_dir=$KERNEL_DIR" >> "$GITHUB_OUTPUT" echo "Cloning $KERNEL_URL @ $KERNEL_REF (shallow)..." git clone --depth 1 --single-branch --branch "$KERNEL_REF" --no-tags \ "$KERNEL_URL" "$KERNEL_DIR" KERNEL_SHA=$(git -C "$KERNEL_DIR" rev-parse HEAD) - echo "KERNEL_SHA=$KERNEL_SHA" >> "$GITHUB_ENV" + echo "kernel_sha=$KERNEL_SHA" >> "$GITHUB_OUTPUT" echo "Kernel HEAD SHA: $KERNEL_SHA" - - name: Derive LOCALVERSION + # Committer date of HEAD, normalised to UTC. Committer rather than + # author date: an author date can be months old on a backported patch, + # which would date the build by when someone first wrote the code + # rather than by when this snapshot came to exist. + KERNEL_DATE=$(TZ=UTC git -C "$KERNEL_DIR" log -1 --format=%cd --date=format-local:%Y%m%d) + [[ "$KERNEL_DATE" =~ ^[0-9]{8}$ ]] || { + echo "::error::could not read a YYYYMMDD commit date from $KERNEL_REF (got '$KERNEL_DATE')" + exit 1 + } + echo "kernel_date=$KERNEL_DATE" >> "$GITHUB_OUTPUT" + echo "Kernel HEAD commit date (UTC): $KERNEL_DATE" + + - name: Derive version fields + id: localversion shell: bash env: LOCALVERSION_INPUT: ${{ inputs.localversion }} FLAVOUR_INPUT: ${{ inputs.flavour }} + KERNEL_REF: ${{ steps.ref.outputs.kernel_ref }} + KERNEL_SHA: ${{ steps.clone.outputs.kernel_sha }} + KERNEL_DATE: ${{ steps.clone.outputs.kernel_date }} run: | set -euo pipefail if [[ -n "$LOCALVERSION_INPUT" ]]; then + # An override supplies no snapshot, and the ref it was chosen to + # replace cannot be trusted to describe it. The Debian version then + # carries no +git; prepare-source.sh warns about the same. echo "Using explicit LOCALVERSION override: $LOCALVERSION_INPUT" - echo "LOCALVERSION=$LOCALVERSION_INPUT" >> "$GITHUB_ENV" + echo "localversion=$LOCALVERSION_INPUT" >> "$GITHUB_OUTPUT" + echo "snapshot=" >> "$GITHUB_OUTPUT" + echo "gitsha=" >> "$GITHUB_OUTPUT" else - LOCALVERSION=$(ci/scripts/derive-localversion.sh \ + # Emits LOCALVERSION=, SNAPSHOT= and GITSHA= lines. They are read + # here rather than recovered from LOCALVERSION later: that string + # also carries a flavour name, and a hex SHA can end in eight digits + # of its own. + # + # --date is only consulted for branch-tip builds, where the ref + # carries no snapshot. It comes from the commit rather than the + # clock, so rebuilding a commit reproduces its version instead of + # inventing a higher one. + FIELDS=$(ci/scripts/derive-localversion.sh \ --flavour "$FLAVOUR_INPUT" \ --ref "$KERNEL_REF" \ - --sha "$(echo "$KERNEL_SHA" | cut -c1-12)") - echo "LOCALVERSION=$LOCALVERSION" >> "$GITHUB_ENV" - echo "Derived LOCALVERSION: $LOCALVERSION" + --sha "$KERNEL_SHA" \ + --date "$KERNEL_DATE") + echo "Derived version fields:" + echo "$FIELDS" + echo "$FIELDS" | while IFS='=' read -r key value; do + case "$key" in + LOCALVERSION) echo "localversion=$value" ;; + SNAPSHOT) echo "snapshot=$value" ;; + GITSHA) echo "gitsha=$value" ;; + *) echo "::error::unexpected field '$key' from derive-localversion.sh"; exit 1 ;; + esac + done >> "$GITHUB_OUTPUT" fi - name: Resolve Debian revision @@ -196,10 +255,12 @@ runs: # exactly the revision the daily build would have used. The lookup is # always the Daily entry: direct dispatch is build-only and never # promotes. + id: revision shell: bash env: DEBIAN_REVISION_INPUT: ${{ inputs.debian-revision }} BUILD_INPUT: ${{ inputs.build }} + DISTRO: ${{ steps.env.outputs.distro }} run: | set -euo pipefail if [[ -n "$DEBIAN_REVISION_INPUT" ]]; then @@ -217,13 +278,14 @@ runs: } echo "Matrix Debian revision for $BUILD_INPUT/$DISTRO: $DEBIAN_REVISION" fi - echo "DEBIAN_REVISION=$DEBIAN_REVISION" >> "$GITHUB_ENV" + echo "debian_revision=$DEBIAN_REVISION" >> "$GITHUB_OUTPUT" - name: Merge qcom-next PR patches if: ${{ inputs.qcom-next-pr != '' }} shell: bash env: QCOM_NEXT_PR: ${{ inputs.qcom-next-pr }} + KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} run: | cd "$KERNEL_DIR" echo "Merging qcom-next PR(s): $QCOM_NEXT_PR" @@ -250,6 +312,7 @@ runs: shell: bash env: KERNEL_TOPICS_PR: ${{ inputs.kernel-topics-pr }} + KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} run: | cd "$KERNEL_DIR" echo "Applying kernel-topics PR patch(es): $KERNEL_TOPICS_PR" @@ -275,6 +338,15 @@ runs: KERNEL_CONFIG_INPUT: ${{ inputs.kernel-config }} DKMS_INPUT: ${{ inputs.dkms }} DEBUG_BUILD_INPUT: ${{ inputs.debug-build }} + KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} + DISTRO: ${{ steps.env.outputs.distro }} + DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} + LOCALVERSION: ${{ steps.localversion.outputs.localversion }} + SNAPSHOT: ${{ steps.localversion.outputs.snapshot }} + DEBIAN_REVISION: ${{ steps.revision.outputs.debian_revision }} + KERNEL_URL: ${{ inputs.kernel-url }} + KERNEL_REF: ${{ steps.ref.outputs.kernel_ref }} + KERNEL_SHA: ${{ steps.clone.outputs.kernel_sha }} run: | # No -x here: keep xtrace off for this step as it assembles the # prepare-source.sh argument list which may include sensitive paths. @@ -288,7 +360,17 @@ runs: --debian-revision "$DEBIAN_REVISION" ) + # The version fields all come from derive-localversion.sh, which + # composed them from the ref and HEAD together. Nothing here appends to + # them: the SHA is already inside LOCALVERSION, and SNAPSHOT and the + # SHA are passed separately for the Debian version, which joins them + # differently. [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") + [[ -n "$SNAPSHOT" ]] && ARGS+=(--snapshot "$SNAPSHOT") + # Full SHA: prepare-source.sh takes the first 12 for the version + # strings and records the whole thing in the changelog, alongside the + # repository and ref that the version strings do not name. + ARGS+=(--git-sha "$KERNEL_SHA" --git-clone "$KERNEL_URL" --git-ref "$KERNEL_REF") [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") # Always passed, unlike the optional inputs above: an empty list is a @@ -303,6 +385,8 @@ runs: - name: Pack prepared source tree shell: bash + env: + KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} run: | set -euo pipefail tar czf /tmp/kernel-srcpkg.tar.gz \ diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index a885bcea..ead20aad 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -203,6 +203,8 @@ jobs: # was dispatched from or the one the calling workflow runs at. - name: Checkout pkg-linux-qcom uses: actions/checkout@v4 + with: + persist-credentials: false - name: Prepare kernel source uses: ./.github/actions/prepare-kernel-source @@ -247,6 +249,8 @@ jobs: # comes from the prepared-source artifact, not from this checkout. - name: Checkout pkg-linux-qcom uses: actions/checkout@v4 + with: + persist-credentials: false - name: Build in Debusine id: debusine @@ -278,6 +282,7 @@ jobs: - name: Checkout debusine-action helpers uses: actions/checkout@v5 with: + persist-credentials: false repository: qualcomm-linux/debusine-action ref: main path: debusine-action diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 1e4a8aa1..94107d93 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -185,6 +185,8 @@ jobs: steps: - name: Checkout pkg-linux-qcom uses: actions/checkout@v4 + with: + persist-credentials: false - name: Prepare kernel source uses: ./.github/actions/prepare-kernel-source @@ -223,12 +225,15 @@ jobs: steps: - name: Checkout pkg-linux-qcom uses: actions/checkout@v4 + with: + persist-credentials: false # Pinned to @main while native kernel-build support is still landing # upstream in qualcomm-linux/docker-pkg-build. - name: Checkout docker-pkg-build uses: actions/checkout@v4 with: + persist-credentials: false repository: qualcomm-linux/docker-pkg-build ref: main path: docker-pkg-build diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index f78b97f3..9a2f8533 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -63,6 +63,8 @@ jobs: ubuntu: ${{ steps.set-matrix.outputs.ubuntu }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Generate build matrix id: set-matrix diff --git a/.github/workflows/release-kernel-debian.yml b/.github/workflows/release-kernel-debian.yml index 759d054b..46ac9836 100644 --- a/.github/workflows/release-kernel-debian.yml +++ b/.github/workflows/release-kernel-debian.yml @@ -110,6 +110,8 @@ jobs: steps: - name: Checkout pkg-linux-qcom uses: actions/checkout@v4 + with: + persist-credentials: false - name: Prepare kernel source uses: ./.github/actions/prepare-kernel-source @@ -150,6 +152,8 @@ jobs: steps: - name: Checkout pkg-linux-qcom uses: actions/checkout@v4 + with: + persist-credentials: false - name: Build in Debusine id: debusine From 2181cb6d90261b1937f8330e8f89ad3fdd1932b8 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 23/71] ci: name the Ubuntu build job for where it builds "Build kernel package" said what every build job in the repository does, so next to "Build (Debusine)" it read as the generic one rather than as the other half of a pair. Both jobs build a kernel package; what separates them is where. Name it "Build (Docker)", which is what the rest of the repository already calls this path -- build-kernel.sh --build-mode docker, the README's Docker path -- and the two now differ by the word that actually differs: qcom-next-forky / Build (Debusine) qcom-next-resolute / Build (Docker) The step inside it keeps the name, as the Debusine steps do; it is the job listing that had the ambiguity. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-ubuntu.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 94107d93..0856befa 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -212,9 +212,13 @@ jobs: # --------------------------------------------------------------------------- # build: build the prepared source in a suite-matched container, then publish. + # + # Named for where the build happens, as the Debian workflows' Build (Debusine) + # is: this one runs in a suite-matched pkg-builder container on the runner + # rather than being submitted to a build service. # --------------------------------------------------------------------------- build: - name: Build kernel package + name: Build (Docker) needs: prepare runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] env: From 93dc87edddaee3ff13f09e06ff82903964690385 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 24/71] ci: pull the builder image instead of rebuilding it every job Every job ran docker_deb_build.py --rebuild, which deletes any existing pkg-builder image and builds a fresh one from docker-pkg-build's Dockerfile. The name it tags looks like a registry reference, ghcr.io/qualcomm-linux/ pkg-builder:, but nothing here ever pushed or pulled it: the docker run that follows was using a local image the previous step had just made. That image is published. docker-pkg-build's container-build-and-upload.yml builds trixie, noble and resolute, validates each by building a test package in it, and pushes them on every push to main and again weekly. So each of our jobs was rebuilding, from the same Dockerfile, an image that upstream had already built and tested -- once for a Debian leg, twice for an Ubuntu one, on ephemeral runners that keep no layer cache between them. Pull trixie and resolute, which covers all eleven daily legs: the ten Debian ones use the trixie image, and resolute is the only Ubuntu one. The package is not public, so the pull logs in with GITHUB_TOKEN under the packages: read permission these workflows already declared and had no use for until now. Suites with no published image still build one on the runner, so the path stays. In prepare-kernel-source both that build and the docker-pkg-build checkout it needs are now conditional, and the Ubuntu build job needs no step at all for them: docker_deb_build.py builds the image itself when it finds it missing, so not pulling is instruction enough. forky is worth a note, having no image of its own in the registry and no Dockerfile in docker-pkg-build to build one from. It uses the trixie image and therefore pulls it, which costs nothing today: Debusine performs the actual suite-specific build, and this container only runs prepare-source.sh, which generates packaging rather than compiling anything. The comment says so, so that the day a forky image appears it is clear what to change. Signed-off-by: Christopher Obbard --- .../actions/prepare-kernel-source/action.yml | 57 ++++++++++++++++--- .github/workflows/build-kernel-ubuntu.yml | 22 ++++++- 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index bfe94518..4996b387 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -27,6 +27,13 @@ inputs: suite: description: 'Target suite' required: true + registry-token: + description: > + Token used to pull the pkg-builder image from ghcr.io, which is not a + public package. Defaults to the job's own GITHUB_TOKEN, which needs the + packages: read permission the calling workflows declare. The secrets + context does not reach a composite action, hence an input. + default: ${{ github.token }} expect-family: description: > Build family the calling workflow builds, debian or ubuntu. The suite is @@ -91,13 +98,17 @@ runs: run: | set -euo pipefail # Family and docker image in one place, because they answer the same - # question about a suite. Debian-family suites build in the trixie - # image: docker-pkg-build and pkg-builder only support trixie for - # Debian, and Debusine handles the actual suite-specific build - # internally. Ubuntu-family suites build in an image of their own - # suite. The family list here is the one in + # question about a suite. The family list here is the one in # ci/scripts/resolve-matrix.py, which routes an entry to the workflow # that calls this action. + # + # Ubuntu-family suites use an image of their own suite. Debian-family + # suites all use the trixie image: the registry publishes no forky + # image, and docker-pkg-build has no Dockerfile to build one from + # either, so forky borrows trixie's until one exists. That costs + # nothing today because Debusine performs the actual suite-specific + # build; this container only has to run prepare-source.sh, which + # generates packaging rather than compiling anything. case "$SUITE_INPUT" in trixie|bookworm) FAMILY=debian; DOCKER_DISTRO="$SUITE_INPUT" ;; forky|sid|unstable) FAMILY=debian; DOCKER_DISTRO=trixie ;; @@ -110,11 +121,41 @@ runs: exit 1 fi + # Pull the images qualcomm-linux/docker-pkg-build publishes, and build + # the rest here. Its container-build-and-upload.yml pushes trixie and + # resolute (and noble) on every push to main and weekly, having + # validated each with a test package build, so rebuilding them from the + # same Dockerfile on every job repeats work already done. Anything it + # does not publish still has to be built on the runner. + case "$DOCKER_DISTRO" in + trixie|resolute) IMAGE_SOURCE=pull ;; + *) IMAGE_SOURCE=build ;; + esac + echo "distro=$SUITE_INPUT" >> "$GITHUB_OUTPUT" echo "docker_distro=$DOCKER_DISTRO" >> "$GITHUB_OUTPUT" - echo "Suite $SUITE_INPUT is $FAMILY-family and builds in the $DOCKER_DISTRO image" + echo "image_source=$IMAGE_SOURCE" >> "$GITHUB_OUTPUT" + echo "Suite $SUITE_INPUT is $FAMILY-family and uses the $DOCKER_DISTRO image ($IMAGE_SOURCE)" + - name: Pull builder image + if: ${{ steps.env.outputs.image_source == 'pull' }} + shell: bash + env: + DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} + REGISTRY_TOKEN: ${{ inputs.registry-token }} + run: | + # No -x: REGISTRY_TOKEN is in env. + set -euo pipefail + # ghcr.io/qualcomm-linux/pkg-builder is not a public package, so the + # pull needs a login even though the image is only ever read. + echo "$REGISTRY_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + docker pull "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" + docker image ls + + # Only for a suite with no published image: the pull path needs neither + # this checkout nor the build below. - name: Checkout docker-pkg-build + if: ${{ steps.env.outputs.image_source == 'build' }} uses: actions/checkout@v4 with: persist-credentials: false @@ -122,11 +163,13 @@ runs: ref: main path: docker-pkg-build - - name: Build docker image + - name: Build builder image + if: ${{ steps.env.outputs.image_source == 'build' }} shell: bash env: DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} run: | + set -euo pipefail ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DOCKER_DISTRO" docker image ls diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 0856befa..33590752 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -242,10 +242,26 @@ jobs: ref: main path: docker-pkg-build - - name: Build docker image + - name: Fetch builder image + env: + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - # --rebuild forces a fresh suite-matched pkg-builder image. - ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DISTRO" + # No -x: REGISTRY_TOKEN is in env. + set -euo pipefail + # Same split as prepare-kernel-source: pull what docker-pkg-build + # publishes, build what it does not. Nothing is forced here, because + # docker_deb_build.py builds the image itself when the build below + # finds it missing -- so an unpublished suite needs no step of its + # own, only the absence of a pull. + case "$DISTRO" in + resolute) + echo "$REGISTRY_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + docker pull "ghcr.io/qualcomm-linux/pkg-builder:$DISTRO" + ;; + *) + echo "No published pkg-builder image for $DISTRO; it will be built on demand." + ;; + esac docker image ls - name: Download prepared source tree From 86ead59ce8768ad8495da01ea259f6b59718fb36 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 25/71] ci: name the jobs for what they produce "Prepare kernel source" and "Build (Debusine)" named a stage and a place, so the pair read as different kinds of thing and neither said what came out of it. Name all three for their product instead, and the sequence states itself in the checks list: qcom-next-forky / Generate source package / Build package (Debusine) / Publish to S3 qcom-next-resolute / Generate source package / Build package (Docker) The parenthesis keeps doing what it did before: source package generation is one job whichever family runs it, and only the binary build differs, so that is the only name that has to say where it happens. Display names only. The job ids are untouched, so needs:, the comments and the README all still refer to prepare, build, publish and release as they did. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 4 ++-- .github/workflows/build-kernel-ubuntu.yml | 10 +++++----- .github/workflows/release-kernel-debian.yml | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index ead20aad..57b2399f 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -192,7 +192,7 @@ jobs: # prepare: clone kernel source, inject packaging, run prepare-source.sh. # --------------------------------------------------------------------------- prepare: - name: Prepare kernel source + name: Generate source package runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] steps: # debian/ and ci/ live on the same branch, so one checkout supplies both @@ -232,7 +232,7 @@ jobs: # build: generate the Debian source package and submit it to Debusine. # --------------------------------------------------------------------------- build: - name: Build (Debusine) + name: Build package (Debusine) needs: prepare runs-on: ubuntu-latest environment: Staging diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 33590752..7bd24b0a 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -180,7 +180,7 @@ jobs: # prepare: clone kernel source, inject packaging, run prepare-source.sh. # --------------------------------------------------------------------------- prepare: - name: Prepare kernel source + name: Generate source package runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] steps: - name: Checkout pkg-linux-qcom @@ -213,12 +213,12 @@ jobs: # --------------------------------------------------------------------------- # build: build the prepared source in a suite-matched container, then publish. # - # Named for where the build happens, as the Debian workflows' Build (Debusine) - # is: this one runs in a suite-matched pkg-builder container on the runner - # rather than being submitted to a build service. + # Named for where the build happens, as the Debian workflows' Build package + # (Debusine) is: this one runs in a suite-matched pkg-builder container on + # the runner rather than being submitted to a build service. # --------------------------------------------------------------------------- build: - name: Build (Docker) + name: Build package (Docker) needs: prepare runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] env: diff --git a/.github/workflows/release-kernel-debian.yml b/.github/workflows/release-kernel-debian.yml index 46ac9836..abe8d060 100644 --- a/.github/workflows/release-kernel-debian.yml +++ b/.github/workflows/release-kernel-debian.yml @@ -105,7 +105,7 @@ jobs: # prepare: clone kernel source, inject packaging, run prepare-source.sh. # --------------------------------------------------------------------------- prepare: - name: Prepare kernel source + name: Generate source package runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] steps: - name: Checkout pkg-linux-qcom @@ -136,7 +136,7 @@ jobs: # build: generate the Debian source package and submit it to Debusine. # --------------------------------------------------------------------------- build: - name: Build (Debusine) + name: Build package (Debusine) needs: prepare runs-on: ubuntu-latest environment: Production From 0d10ee0830303ce05c2116530e03a9f1bbd2bc8e Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 26/71] ci: drop the questing and bookworm suites from CI Both were listed as buildable and neither was. questing is skipped by docker_deb_build.py's SKIP_REBUILD_DISTROS and has no published image, so a questing run would build nothing and then fail pulling one. bookworm has no Dockerfile in docker-pkg-build at all, and no published image either. Neither appears in ci/build-matrix.yaml, so both failures were only ever waiting for someone to dispatch them by hand -- which build-kernel-ubuntu.yml invited by naming questing in its suite description. bookworm goes further than the description: it was in resolve-matrix.py's DEBIAN_SUITES, which decides which workflow builds an entry, and it was the only Debian suite mapping to an image of its own name. Without it every Debian-family suite maps to trixie, so the two case arms that did that become one arm saying so. The packaging keeps questing. prepare-source.sh accepts it in VALID_DISTROS and debian/README.md lists it as a supported distribution, which is a claim about the packaging rather than about CI, and it is true: what is missing is a container image to build it in, not support for the suite. bookworm was never in either, so nothing there to remove. Signed-off-by: Christopher Obbard --- .../actions/prepare-kernel-source/action.yml | 20 +++++++++---------- .github/workflows/build-kernel-ubuntu.yml | 4 ++-- ci/scripts/resolve-matrix.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index 4996b387..d109e5a5 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -102,17 +102,17 @@ runs: # ci/scripts/resolve-matrix.py, which routes an entry to the workflow # that calls this action. # - # Ubuntu-family suites use an image of their own suite. Debian-family - # suites all use the trixie image: the registry publishes no forky - # image, and docker-pkg-build has no Dockerfile to build one from - # either, so forky borrows trixie's until one exists. That costs - # nothing today because Debusine performs the actual suite-specific - # build; this container only has to run prepare-source.sh, which - # generates packaging rather than compiling anything. + # Ubuntu-family suites use an image of their own suite. Every + # Debian-family suite uses the trixie image, forky and sid included: + # the registry publishes no image for either, and docker-pkg-build has + # no Dockerfile to build one from, so they borrow trixie's until one + # exists. That costs nothing today because Debusine performs the actual + # suite-specific build; this container only has to run + # prepare-source.sh, which generates packaging rather than compiling + # anything. case "$SUITE_INPUT" in - trixie|bookworm) FAMILY=debian; DOCKER_DISTRO="$SUITE_INPUT" ;; - forky|sid|unstable) FAMILY=debian; DOCKER_DISTRO=trixie ;; - *) FAMILY=ubuntu; DOCKER_DISTRO="$SUITE_INPUT" ;; + trixie|forky|sid|unstable) FAMILY=debian; DOCKER_DISTRO=trixie ;; + *) FAMILY=ubuntu; DOCKER_DISTRO="$SUITE_INPUT" ;; esac if [[ "$FAMILY" != "$EXPECT_FAMILY" ]]; then diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 7bd24b0a..1499a35d 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -28,7 +28,7 @@ on: type: string default: 'qcom-next' suite: - description: 'Target Ubuntu suite (noble, questing, resolute)' + description: 'Target Ubuntu suite (noble, resolute)' type: string default: 'resolute' kernel-branch: @@ -102,7 +102,7 @@ on: default: 'qcom-next' type: string suite: - description: 'Target Ubuntu suite (noble, questing, resolute). A Debian suite here is rejected before anything is built; dispatch build-kernel-debian for those.' + description: 'Target Ubuntu suite (noble, resolute). A Debian suite here is rejected before anything is built; dispatch build-kernel-debian for those.' required: true default: 'resolute' type: string diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index 8151cfdf..edadb6a9 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -93,7 +93,7 @@ # runners, containers and publish steps, so a caller selects entries by family # and calls the one workflow that builds them -- rather than every build leg # starting both and skipping one. -DEBIAN_SUITES = ("trixie", "forky", "sid", "unstable", "bookworm") +DEBIAN_SUITES = ("trixie", "forky", "sid", "unstable") FAMILIES = ("debian", "ubuntu") From d6485824c3396c5866530e2ac120a8bdb93f643d Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 27/71] ci: pin actions/checkout and actions/upload-artifact to v7.0.1 Both were floating on a major-version tag, and two checkouts were already pinned to a commit while thirteen others were not, so the same action ran at whatever main had moved to depending on which file invoked it. A tag is a mutable ref: it can be repointed at any commit, which is the supply-chain hole pinning closes. Pin every use to the commit the release tag names, with the version in a comment so the next reader knows what is pinned without resolving a SHA: actions/checkout 3d3c42e5aac5ba805825da76410c181273ba90b1 v7.0.1 actions/upload-artifact 043fb46d1a93c77aae656e7c1c64a875d1fc6a0a v7.0.1 Both SHAs were checked against the tags they claim to be rather than taken on trust; each is the commit v7 and v7.0.1 both point at today. This raises a runner requirement. Both actions run on Node.js 24 from v6, which needs Actions Runner 2.327.1 or newer, and the prepare and build jobs run on the self-hosted arm64 runners. Those need to be at that version before this merges, or every job that checks out will fail on them. Signed-off-by: Christopher Obbard --- .github/actions/debusine-build/action.yml | 2 +- .github/actions/prepare-kernel-source/action.yml | 4 ++-- .github/workflows/build-kernel-debian.yml | 6 +++--- .github/workflows/build-kernel-ubuntu.yml | 6 +++--- .github/workflows/build-kernel.yml | 4 ++-- .github/workflows/daily.yml | 2 +- .github/workflows/pr-build.yml | 2 +- .github/workflows/release-kernel-debian.yml | 6 +++--- .github/workflows/release.yml | 2 +- 9 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/actions/debusine-build/action.yml b/.github/actions/debusine-build/action.yml index 6355aea6..4c660436 100644 --- a/.github/actions/debusine-build/action.yml +++ b/.github/actions/debusine-build/action.yml @@ -64,7 +64,7 @@ runs: using: composite steps: - name: Checkout debusine-action helpers - uses: actions/checkout@v5 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false repository: qualcomm-linux/debusine-action diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index d109e5a5..1aba18a2 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -156,7 +156,7 @@ runs: # this checkout nor the build below. - name: Checkout docker-pkg-build if: ${{ steps.env.outputs.image_source == 'build' }} - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false repository: qualcomm-linux/docker-pkg-build @@ -437,7 +437,7 @@ runs: "$(basename "$KERNEL_DIR")" - name: Upload prepared source tree artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} path: /tmp/kernel-srcpkg.tar.gz diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 57b2399f..49d390df 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -202,7 +202,7 @@ jobs: # An empty ref leaves actions/checkout on github.sha, the commit this run # was dispatched from or the one the calling workflow runs at. - name: Checkout pkg-linux-qcom - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -248,7 +248,7 @@ jobs: # Only for the debusine-build action itself: the package being built # comes from the prepared-source artifact, not from this checkout. - name: Checkout pkg-linux-qcom - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -280,7 +280,7 @@ jobs: shell: bash steps: - name: Checkout debusine-action helpers - uses: actions/checkout@v5 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false repository: qualcomm-linux/debusine-action diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 1499a35d..e3042831 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -184,7 +184,7 @@ jobs: runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] steps: - name: Checkout pkg-linux-qcom - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -228,14 +228,14 @@ jobs: shell: bash steps: - name: Checkout pkg-linux-qcom - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false # Pinned to @main while native kernel-build support is still landing # upstream in qualcomm-linux/docker-pkg-build. - name: Checkout docker-pkg-build - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false repository: qualcomm-linux/docker-pkg-build diff --git a/.github/workflows/build-kernel.yml b/.github/workflows/build-kernel.yml index aa37b234..e0326745 100644 --- a/.github/workflows/build-kernel.yml +++ b/.github/workflows/build-kernel.yml @@ -43,7 +43,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Extract org, repo and branch name run: | @@ -57,7 +57,7 @@ jobs: echo "REPO_NAME=$REPO_NAME" - name: Checkout qcom-build-utils - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: qualcomm-linux/qcom-build-utils ref: ${{ inputs.qcom-build-utils-ref }} diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 9a2f8533..ef184b80 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -62,7 +62,7 @@ jobs: debian: ${{ steps.set-matrix.outputs.debian }} ubuntu: ${{ steps.set-matrix.outputs.ubuntu }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 7b3f699a..ed8f0c71 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -33,7 +33,7 @@ jobs: debian: ${{ steps.set-matrix.outputs.debian }} ubuntu: ${{ steps.set-matrix.outputs.ubuntu }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} persist-credentials: false diff --git a/.github/workflows/release-kernel-debian.yml b/.github/workflows/release-kernel-debian.yml index abe8d060..3b1ad19d 100644 --- a/.github/workflows/release-kernel-debian.yml +++ b/.github/workflows/release-kernel-debian.yml @@ -109,7 +109,7 @@ jobs: runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] steps: - name: Checkout pkg-linux-qcom - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -151,7 +151,7 @@ jobs: srcpkg_version: ${{ steps.debusine.outputs.srcpkg_version }} steps: - name: Checkout pkg-linux-qcom - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -189,7 +189,7 @@ jobs: shell: bash steps: - name: Checkout debusine-action helpers - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false repository: qualcomm-linux/debusine-action diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bdbbb1e1..b31fc582 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,7 +51,7 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false From ed939016c38947a82e7787cc91784cb59f0a4cac Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 28/71] ci: scope the token per job and say what each permission is for Every workflow granted contents: read and packages: read to all of its jobs, so a job that only reads the repository still held a token that could read packages, and nothing recorded which job needed which. Give each job the permissions it uses and name the consumer in a comment, so the next person to add a step can tell whether the token already covers it: permissions: contents: read # actions/checkout packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder Publish to S3 is the one job that narrows: it checks out debusine-action and reads the built packages out of Debusine over HTTP, and pulls no image, so it keeps contents: read alone. Every other job either pulls the pkg-builder image or runs in the debusine-pkg-builder container, and the comment says which. Job-level permissions replace the workflow-level ones rather than adding to them, so each block lists everything its job needs. On the jobs that call a reusable workflow the block is a ceiling for the whole call, which is why daily.yml, pr-build.yml and release.yml grant packages: read there: the workflow they call pulls images even though the calling job runs nothing itself. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 8 ++++++++ .github/workflows/build-kernel-ubuntu.yml | 6 ++++++ .github/workflows/daily.yml | 8 ++++++++ .github/workflows/pr-build.yml | 8 ++++++++ .github/workflows/release-kernel-debian.yml | 9 +++++++++ .github/workflows/release.yml | 5 +++++ 6 files changed, 44 insertions(+) diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 49d390df..3bf3b4bf 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -194,6 +194,9 @@ jobs: prepare: name: Generate source package runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder steps: # debian/ and ci/ live on the same branch, so one checkout supplies both # the packaging and the scripts, the delivery matrix they read, and the @@ -235,6 +238,9 @@ jobs: name: Build package (Debusine) needs: prepare runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout + packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder environment: Staging container: image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie @@ -275,6 +281,8 @@ jobs: name: Publish to S3 needs: build runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout defaults: run: shell: bash diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index e3042831..f09e3ba4 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -182,6 +182,9 @@ jobs: prepare: name: Generate source package runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder steps: - name: Checkout pkg-linux-qcom uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -221,6 +224,9 @@ jobs: name: Build package (Docker) needs: prepare runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder env: DISTRO: ${{ inputs.suite }} defaults: diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index ef184b80..12eda419 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -58,6 +58,8 @@ jobs: configure-matrix: name: Configure build matrix runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout outputs: debian: ${{ steps.set-matrix.outputs.debian }} ubuntu: ${{ steps.set-matrix.outputs.ubuntu }} @@ -137,6 +139,9 @@ jobs: fail-fast: false matrix: include: ${{ fromJson(needs.configure-matrix.outputs.debian) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls uses: ./.github/workflows/build-kernel-debian.yml with: suite: ${{ matrix.suite }} @@ -166,6 +171,9 @@ jobs: fail-fast: false matrix: include: ${{ fromJson(needs.configure-matrix.outputs.ubuntu) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls uses: ./.github/workflows/build-kernel-ubuntu.yml with: suite: ${{ matrix.suite }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index ed8f0c71..8dfeea50 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -29,6 +29,8 @@ jobs: configure-matrix: name: Configure build matrix runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout outputs: debian: ${{ steps.set-matrix.outputs.debian }} ubuntu: ${{ steps.set-matrix.outputs.ubuntu }} @@ -74,6 +76,9 @@ jobs: fail-fast: false matrix: include: ${{ fromJson(needs.configure-matrix.outputs.debian) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls uses: ./.github/workflows/build-kernel-debian.yml with: suite: ${{ matrix.suite }} @@ -104,6 +109,9 @@ jobs: fail-fast: false matrix: include: ${{ fromJson(needs.configure-matrix.outputs.ubuntu) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls uses: ./.github/workflows/build-kernel-ubuntu.yml with: suite: ${{ matrix.suite }} diff --git a/.github/workflows/release-kernel-debian.yml b/.github/workflows/release-kernel-debian.yml index 3b1ad19d..f4eeb40a 100644 --- a/.github/workflows/release-kernel-debian.yml +++ b/.github/workflows/release-kernel-debian.yml @@ -107,6 +107,9 @@ jobs: prepare: name: Generate source package runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] + permissions: + contents: read # actions/checkout + packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder steps: - name: Checkout pkg-linux-qcom uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -139,6 +142,9 @@ jobs: name: Build package (Debusine) needs: prepare runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout + packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder environment: Production container: image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie @@ -177,6 +183,9 @@ jobs: name: Release to Debusine needs: build runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout + packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder environment: Production container: image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b31fc582..a96f4caa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,6 +48,8 @@ jobs: configure-matrix: name: Configure release matrix runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: @@ -105,6 +107,9 @@ jobs: fail-fast: false matrix: include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls uses: ./.github/workflows/release-kernel-debian.yml with: suite: ${{ matrix.suite }} From 0e2461cb9241c3e7e2d678c2dbe4a2d1ec670e91 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 29/71] ci: pin actions/download-artifact to v8.0.1 Four majors behind, and floating on a tag. It is the counterpart of the upload-artifact pinned alongside it: prepare uploads the prepared source tree, and both build paths download it again. v8's inputs are a superset of v4's, adding digest-mismatch and skip-decompress and dropping nothing, so the name and path this repository passes still mean what they meant. Like the actions pinned before it, v8 runs on Node.js 24 and so needs Actions Runner 2.327.1 or newer. Both callers are self-hosted jobs. Signed-off-by: Christopher Obbard --- .github/actions/debusine-build/action.yml | 2 +- .github/workflows/build-kernel-ubuntu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/debusine-build/action.yml b/.github/actions/debusine-build/action.yml index 4c660436..e089dff9 100644 --- a/.github/actions/debusine-build/action.yml +++ b/.github/actions/debusine-build/action.yml @@ -75,7 +75,7 @@ runs: lib - name: Download prepared source tree - uses: actions/download-artifact@v4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} path: /tmp/srcpkg-artifact diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index f09e3ba4..e89f7fab 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -271,7 +271,7 @@ jobs: docker image ls - name: Download prepared source tree - uses: actions/download-artifact@v4 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} path: /tmp/srcpkg-artifact From f874e2188c030fc6b15def4177442ca2cffa9764 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 30/71] ci: pin actions/stale to v11.0.0 Two majors behind on a floating tag. All nine inputs this workflow sets still exist in v11, so the 60-day stale and 10-day close policy it configures is carried over unchanged. v10 moved to Node.js 24 and wants Actions Runner 2.327.1 or newer, which costs nothing here: this is the one workflow that runs on ubuntu-latest rather than on the self-hosted runners. Signed-off-by: Christopher Obbard --- .github/workflows/stale-issues.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale-issues.yaml b/.github/workflows/stale-issues.yaml index e84b54b4..07530ad4 100644 --- a/.github/workflows/stale-issues.yaml +++ b/.github/workflows/stale-issues.yaml @@ -11,7 +11,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v9 + - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0 with: stale-issue-message: 'This issue has been marked as stale due to 60 days of inactivity. To prevent automatic closure in 10 days, remove the stale label or add a comment. You can reopen a closed issue at any time.' stale-pr-message: 'This pull request has been marked as stale due to 60 days of inactivity. To prevent automatic closure in 10 days, remove the stale label or add a comment. You can reopen a closed pull request at any time.' From c2a80c41d0e4475059b77a70c77d6596bb6ec9f6 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 31/71] ci: pin upload-private-artifact-action to the aws-v4 commit Pinned, not bumped: aws-v5 exists but this stays on aws-v4. The two new workflows named the aws-v4 tag, which is at least a tag. The legacy build-kernel.yml named aws, which is a branch, so it ran whatever had last landed there -- the mutable ref this repository otherwise no longer has, and one publishing artifacts to S3 at that. All three now name the commit aws-v4 points at, so the branch stops deciding what the legacy workflow runs and all three uses agree on one version. For build-kernel.yml that settles it at aws-v4 rather than at the branch head, which currently carries the aws-v5 interface. Nothing it passes is affected: s3_bucket, path and destination are inputs of both, and the inputs aws-v5 adds are ones no caller here sets. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 2 +- .github/workflows/build-kernel-ubuntu.yml | 2 +- .github/workflows/build-kernel.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 3bf3b4bf..e6dcf24d 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -334,7 +334,7 @@ jobs: # Keep flavour and suite in the destination so concurrent matrix legs # cannot overwrite or mix package outputs. Consumers must select the # flavour and suite they intend to install. - uses: qualcomm-linux/upload-private-artifact-action@aws-v4 + uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 with: s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} path: deb-artifacts diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index e89f7fab..270c15b3 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -305,7 +305,7 @@ jobs: # Keep flavour and suite in the destination so concurrent matrix legs # cannot overwrite or mix package outputs. Consumers must select the # flavour and suite they intend to install. - uses: qualcomm-linux/upload-private-artifact-action@aws-v4 + uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 with: s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} path: kernel-build/${{ inputs.suite }} diff --git a/.github/workflows/build-kernel.yml b/.github/workflows/build-kernel.yml index e0326745..120e66ba 100644 --- a/.github/workflows/build-kernel.yml +++ b/.github/workflows/build-kernel.yml @@ -276,7 +276,7 @@ jobs: ' - name: Upload kernel .deb package to S3 - uses: qualcomm-linux/upload-private-artifact-action@aws + uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 with: s3_bucket: qli-prd-lecore-gh-artifacts path: qcom-build-utils/kernel/deb_artifact From 976b892153126a6c1eb358f43b196abfae900a84 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 32/71] ci: add the Qualcomm preflight checks workflow .github/workflows/Readme.md has listed this workflow as one of the two this repository runs for some time, and it was the one that was not here. Copied from qualcomm-linux/pkg-tftp-server, where it is already in use, so every repository running it runs the same file. It calls qualcomm/qcom-reusable-workflows' preflight orchestrator, enabling the semgrep scan, dependency review, repolinter, copyright and licence check and commit email check, and leaving the commit message check and armor checkers off. It runs on pull requests to main, on push to main, and on dispatch. security-events: write is what lets the semgrep results reach code scanning; the rest is read. Taken verbatim, including its @v2 reference to the orchestrator, which is the one floating ref left in this repository now that every action is pinned to a commit. That is deliberate here and worth stating: a compliance check is wanted at its current definition rather than frozen at the rules of the day it was added, and pinning it would quietly stop this repository receiving new checks. It is a Qualcomm-owned reusable workflow rather than a third-party action, and holding the file identical to its siblings is what makes it updatable everywhere at once. Signed-off-by: Christopher Obbard --- .github/workflows/qcom-preflight-checks.yml | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/qcom-preflight-checks.yml diff --git a/.github/workflows/qcom-preflight-checks.yml b/.github/workflows/qcom-preflight-checks.yml new file mode 100644 index 00000000..51b7aca4 --- /dev/null +++ b/.github/workflows/qcom-preflight-checks.yml @@ -0,0 +1,24 @@ +name: Qualcomm Preflight Checks +on: + pull_request: + branches: [ main ] + push: + branches: [ main ] + workflow_dispatch: + +permissions: + contents: read + security-events: write + +jobs: + preflight: + name: Run QC Preflight Checks + uses: qualcomm/qcom-reusable-workflows/.github/workflows/reusable-qcom-preflight-checks-orchestrator.yml@v2 + with: + enable-semgrep-scan: true + enable-dependency-review: true + enable-repolinter-check: true + enable-copyright-license-check: true + enable-commit-email-check: true + enable-commit-msg-check: false + enable-armor-checkers: false From 077e284b5281158cdd40591e59324e9dd8116798 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 33/71] ci: remove the deprecated build-kernel workflow Deprecated in so many words by 8a9e784, which took the nightly cron off it and gave that schedule to build-kernel-deb, leaving this one dispatch-only and waiting for somebody to notice. Its replacement has since been replaced again: build-kernel-deb split into build-kernel-debian and build-kernel-ubuntu, so the path it handed over to is two generations on. Nothing referenced it. It was the last caller of the qcom-build-utils build system -- its own kmake image from artifacts.codelinaro.org, its own build_kernel.sh and build-kernel-deb.sh, its own S3 layout under pkg/temp without a flavour in the path -- none of which any current workflow uses. It was also the last thing needing DEB_PKG_BOT_CI_TOKEN, and the last file in .github with unpinned actions before the pinning commits reached it, an invalid secrets: key under workflow_dispatch that actionlint rejects, and no permissions: block at all. build-kernel.sh stays. It is the packaging entry point build-kernel-ubuntu.yml runs, and shares nothing with this file but a name. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel.yml | 283 ----------------------------- 1 file changed, 283 deletions(-) delete mode 100644 .github/workflows/build-kernel.yml diff --git a/.github/workflows/build-kernel.yml b/.github/workflows/build-kernel.yml deleted file mode 100644 index 120e66ba..00000000 --- a/.github/workflows/build-kernel.yml +++ /dev/null @@ -1,283 +0,0 @@ -name: build-kernel - -# Controls when the workflow will run -on: - workflow_dispatch: - inputs: - qcom-build-utils-ref: - description: qcom-build-utils ref branch or commit - required: false - default: 'main' - kernel-branch: - description: Kernel Branch or Tag to sync - required: false - default: 'qcom-next' - kernel-url: - description: Custom Kernel Repo URL (Leave empty for default qualcomm-linux/kernel) - required: false - default: '' - qcom-next-pr: - description: Space-separated PR numbers to merge from qcom-next - type: string - required: false - default: '' - kernel-topics-pr: - description: Space-separated list of kernel-topics PR numbers - type: string - required: false - default: '' - - secrets: - DEB_PKG_BOT_CI_TOKEN: - required: true - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] - - env: - KERNEL_BRANCH: ${{ github.event.inputs.kernel-branch || 'qcom-next' }} - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Extract org, repo and branch name - run: | - # Extract org (owner) and repo name from GITHUB_REPOSITORY (format: org/repo) - echo "ORG_NAME=${GITHUB_REPOSITORY%%/*}" >> "$GITHUB_ENV" - echo "REPO_NAME=${GITHUB_REPOSITORY#*/}" >> "$GITHUB_ENV" - - - name: Debug print exported env - run: | - echo "ORG_NAME=$ORG_NAME" - echo "REPO_NAME=$REPO_NAME" - - - name: Checkout qcom-build-utils - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: qualcomm-linux/qcom-build-utils - ref: ${{ inputs.qcom-build-utils-ref }} - token: ${{ secrets.DEB_PKG_BOT_CI_TOKEN }} - path: ./qcom-build-utils - fetch-depth: 1 - - - name: Deduce qcom-build-utils head commit SHA - run: | - # Get the tip commit SHA of the qcom-build-utils branch/tag used - cd qcom-build-utils - QCOM_BUILD_UTILS_SHA=$(git rev-parse HEAD) - echo "QCOM_BUILD_UTILS_SHA=$QCOM_BUILD_UTILS_SHA" >> "$GITHUB_ENV" - # Echo for logs - echo "Head SHA for qcom-build-utils: $QCOM_BUILD_UTILS_SHA" - - - name: Pull docker image - run: | - docker pull artifacts.codelinaro.org/clo-420-qli-registry/kmake-image-ubuntu-noble-arm64:ver.1.0 - docker tag artifacts.codelinaro.org/clo-420-qli-registry/kmake-image-ubuntu-noble-arm64:ver.1.0 kmake-image:ubuntu-noble-arm64 - - # Runs if kernel-url is EMPTY - - name: Sync kernel codebase (Default) - if: ${{ inputs.kernel-url == '' }} - env: - PAT: ${{ secrets.PAT }} - run: | - cd qcom-build-utils - cd kernel && export BUILD_TOP=`pwd` - echo "build_top=$BUILD_TOP" >> "$GITHUB_ENV" - - export REPO="qualcomm-linux/kernel" - echo "kernel_repo=$REPO" >> "$GITHUB_ENV" - echo "Syncing ${REPO} ref ${KERNEL_BRANCH}" - - git ls-remote --exit-code --heads "https://${PAT}@github.com/${REPO}.git" ${KERNEL_BRANCH} && IS_BRANCH=true || IS_BRANCH=false - - git init $BUILD_TOP/qcom-kernel - cd $BUILD_TOP/qcom-kernel - git remote add origin "https://${PAT}@github.com/${REPO}.git" - - if [ "$IS_BRANCH" = true ]; then - echo "Detected branch: ${KERNEL_BRANCH}" - git fetch --depth=1 origin ${KERNEL_BRANCH} - git checkout ${KERNEL_BRANCH} - else - echo "Assuming tag: ${KERNEL_BRANCH}" - git fetch --depth=1 origin tag ${KERNEL_BRANCH} - git checkout FETCH_HEAD - fi - - # Get Head SHA of kernel branch/tag cloned - QCOM_KERNEL_SHA=$(git rev-parse HEAD) - echo "QCOM_KERNEL_SHA=$QCOM_KERNEL_SHA" >> "$GITHUB_ENV" - - # Runs if kernel-url is PROVIDED - - name: Sync custom kernel codebase - if: ${{ inputs.kernel-url != '' }} - env: - KERNEL_URL_INPUT: ${{ inputs.kernel-url }} - run: | - cd qcom-build-utils - mkdir -p kernel - cd kernel && export BUILD_TOP=`pwd` - echo "build_top=$BUILD_TOP" >> "$GITHUB_ENV" - - KERNEL_URL="$KERNEL_URL_INPUT" - echo "Syncing custom repo: ${KERNEL_URL} ref ${KERNEL_BRANCH}" - - git ls-remote --exit-code --heads ${KERNEL_URL} ${KERNEL_BRANCH} && IS_BRANCH=true || IS_BRANCH=false - - mkdir -p $BUILD_TOP/qcom-kernel - cd $BUILD_TOP/qcom-kernel - git init - git remote add origin ${KERNEL_URL} - - if [ "$IS_BRANCH" = true ]; then - echo "Detected branch: ${KERNEL_BRANCH}" - git fetch --depth=1 origin ${KERNEL_BRANCH} - git checkout ${KERNEL_BRANCH} - else - echo "Assuming tag: ${KERNEL_BRANCH}" - git fetch --depth=1 origin tag ${KERNEL_BRANCH} - git checkout FETCH_HEAD - fi - - QCOM_KERNEL_SHA=$(git rev-parse HEAD) - echo "QCOM_KERNEL_SHA=$QCOM_KERNEL_SHA" >> "$GITHUB_ENV" - - - name: Merge qcom-next PR patches - if: ${{ inputs.kernel-url == '' }} - env: - QCOM_NEXT_PR_INPUT: ${{ github.event.inputs.qcom-next-pr }} - PAT: ${{ secrets.PAT }} - run: | - cd "$build_top/" - if [ -z "$QCOM_NEXT_PR_INPUT" ]; then - echo "No PR number specified, skipping merge." - else - echo "Merging PR(s) #$QCOM_NEXT_PR_INPUT" - - cd "$build_top/qcom-kernel" - - for pr in $QCOM_NEXT_PR_INPUT; do - echo "::group::Merging qcom-next PR #$pr" - git fetch "https://${PAT}@github.com/${kernel_repo}.git" "pull/$pr/head:pr-$pr" - - if ! git merge "pr-$pr" --no-commit; then - echo "Merge conflict while merging PR #$pr. Aborting merge and failing the job." - git merge --abort || true - exit 1 - fi - - if ! git diff --cached --quiet; then - git commit -m "Merged PR #$pr" - else - echo "Nothing to commit for PR #$pr (already merged or fast-forwarded)." - git merge --abort 2>/dev/null || true - fi - echo "::endgroup::" - done - fi - - - name: Apply kernel-topics PR patches - if: ${{ inputs.kernel-url == '' }} - env: - KERNEL_TOPICS_PR_INPUT: ${{ github.event.inputs.kernel-topics-pr }} - run: | - cd "$build_top/" - if [ -z "$KERNEL_TOPICS_PR_INPUT" ]; then - echo "Nothing to apply in kernel topics." - else - echo "Applying PR(s) from topics branch #$KERNEL_TOPICS_PR_INPUT" - for pr in $KERNEL_TOPICS_PR_INPUT; do - echo "::group::Applying kernel-topics PR #$pr" - wget -q "https://github.com/qualcomm-linux/kernel-topics/pull/$pr.patch" -O "$pr.patch" - - - cd "$build_top/qcom-kernel" - - if ! git am "../$pr.patch"; then - echo "Patch application failed for PR #$pr. Aborting 'git am' and failing the job." - git am --abort || true - exit 1 - fi - echo "::endgroup::" - cd "$build_top/" - done - fi - - - name: Enable Kernel Configs - run: | - cd qcom-build-utils/kernel - export BUILD_TOP=$(pwd) - ./scripts/enable_squashfs_configs.sh $BUILD_TOP/qcom-kernel/ - - - name: Run build script - run: | - docker run -i \ - --privileged --rm -v $PWD:$PWD --workdir="$PWD" kmake-image:ubuntu-noble-arm64 \ - -c ' - cd qcom-build-utils/kernel - export BUILD_TOP=`pwd` - ./scripts/build_kernel.sh $BUILD_TOP/qcom-kernel/ - ' - - - name: Build Kernel Debian Package - env: - JOB_ID: ${{ github.run_id }} - JOB_ATTEMPT: ${{ github.run_attempt }} - run: | - docker run -i \ - --privileged --rm -v $PWD:$PWD --workdir="$PWD" \ - -e JOB_ID -e JOB_ATTEMPT \ - kmake-image:ubuntu-noble-arm64 \ - -c ' - cd qcom-build-utils/kernel - export BUILD_TOP=`pwd` - ./scripts/build-kernel-deb.sh out/ ${JOB_ID}-${JOB_ATTEMPT} - mkdir -p deb_artifact - cp ./*.deb deb_artifact/ - ' - - - name: Generate build_info metadata - env: - JOB_ID: ${{ github.run_id }} - JOB_ATTEMPT: ${{ github.run_attempt }} - QCOM_BUILD_UTILS_REF_INPUT: ${{ inputs.qcom-build-utils-ref }} - KERNEL_URL_INPUT: ${{ inputs.kernel-url }} - KERNEL_BRANCH_INPUT: ${{ inputs.kernel-branch }} - QCOM_NEXT_PR_INPUT: ${{ inputs.qcom-next-pr }} - KERNEL_TOPICS_PR_INPUT: ${{ inputs.kernel-topics-pr }} - run: | - docker run -i \ - --privileged --rm -v $PWD:$PWD --workdir="$PWD" \ - -e JOB_ID -e JOB_ATTEMPT -e ORG_NAME -e REPO_NAME \ - -e QCOM_BUILD_UTILS_REF_INPUT -e QCOM_BUILD_UTILS_SHA \ - -e KERNEL_URL_INPUT -e KERNEL_BRANCH_INPUT -e QCOM_KERNEL_SHA \ - -e QCOM_NEXT_PR_INPUT -e KERNEL_TOPICS_PR_INPUT \ - kmake-image:ubuntu-noble-arm64 \ - -c ' - cd qcom-build-utils/kernel - { - echo "JOB_ID: $JOB_ID" - echo "JOB_ATTEMPT: $JOB_ATTEMPT" - echo "ORG_NAME: $ORG_NAME" - echo "REPO_NAME: $REPO_NAME" - echo "QCOM-BUILD-UTILS BRANCH/TAG: $QCOM_BUILD_UTILS_REF_INPUT" - echo "QCOM-BUILD-UTILS HEAD SHA: $QCOM_BUILD_UTILS_SHA" - echo "CUSTOM KERNEL URL: $KERNEL_URL_INPUT" - echo "KERNEL BRANCH/TAG: $KERNEL_BRANCH_INPUT" - echo "KERNEL HEAD SHA: $QCOM_KERNEL_SHA" - echo "PRs FROM QCOM-NEXT: $QCOM_NEXT_PR_INPUT" - echo "PRs FROM KERNEL TOPICS: $KERNEL_TOPICS_PR_INPUT" - } > deb_artifact/build_info - ' - - - name: Upload kernel .deb package to S3 - uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 - with: - s3_bucket: qli-prd-lecore-gh-artifacts - path: qcom-build-utils/kernel/deb_artifact - destination: ${{ env.ORG_NAME }}/pkg/temp/${{ env.REPO_NAME }}/${{ github.run_id }}-${{ github.run_attempt }}/ From b4dd88edd45a170ed3fc9a90e45f1a8f06bb3cfc Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 34/71] ci: dispatch builds by name, defaulting to the whole matrix Every manual build already had to name something the matrix defines, but each workflow asked for it differently, and the two build workflows asked for things the matrix had long since taken over. daily.yml offered a Build scope dropdown whose three options -- full matrix, named builds, one flavour -- were three ways of writing a set of entries, one of which needed a second input to say which set. release.yml offered the same idea under a different name and a different default. build-kernel-debian.yml and build-kernel-ubuntu.yml offered neither: their dispatch predates one entry being one package, and still asked a person to type git-clone, srcpkg, binpkg, kernel-config, dkms and a Debian revision, with defaults frozen at whatever qcom-next/trixie looked like when the form was written. Dispatching one of those built whatever the form said, not what the matrix says, and nothing reconciled the two. So there is now one input, in the two workflows that resolve a matrix: builds, defaulting to all, otherwise a comma-separated list of names. A scheduled run carries no inputs and lands on the same default, which is what it wanted anyway. Selecting a flavour across its suites is naming its entries; the --flavour filter stays in resolve-matrix.py for local use. A name matching no entry of that delivery type still fails the run with the names that do, so a typo cannot quietly narrow a release. daily.yml is the manual entry point that build-kernel-* stops being. It resolves the selection into both families, so one dispatch can name Debian and Ubuntu builds together and each is called by the workflow that builds it -- which is the thing a dispatch of a single-family workflow could never do. The build workflows are workflow_call only now: one run of either is one matrix entry, and a reusable workflow cannot call itself to fan out over a list. The three inputs they offered that the matrix deliberately does not carry -- debug-build, qcom-next-pr, kernel-topics-pr -- move to daily.yml, where they apply to every selected build. They describe a one-off validation run rather than a delivery target, which is why they are inputs and not entry fields. The rest of what that form asked for is gone: it is in the entry, and typing it again was the way to disagree with it. resolve-matrix.py now rejects a build named all, which the dispatch form would otherwise read as every entry, leaving that build unreachable by name. Signed-off-by: Christopher Obbard --- .../actions/prepare-kernel-source/action.yml | 12 +- .github/workflows/build-kernel-debian.yml | 90 +------------ .github/workflows/build-kernel-ubuntu.yml | 89 +------------ .github/workflows/daily.yml | 95 +++++++------- .github/workflows/release.yml | 50 +++---- README.md | 122 +++++++++--------- ci/build-matrix.yaml | 3 +- ci/scripts/resolve-matrix.py | 12 ++ 8 files changed, 157 insertions(+), 316 deletions(-) diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index 1aba18a2..9a993ad2 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -292,12 +292,12 @@ runs: fi - name: Resolve Debian revision - # Callers driven by the matrix pass the entry's own debian-revision - # straight through. A direct workflow_dispatch that leaves it empty looks - # the same entry up itself, so a manual build of a configured build gets - # exactly the revision the daily build would have used. The lookup is - # always the Daily entry: direct dispatch is build-only and never - # promotes. + # The build workflows pass the matrix entry's own debian-revision + # straight through, so this lookup is the fallback for a caller that has + # no entry in hand and leaves the input empty: it looks up the Daily + # entry for the build name and suite it was given, and builds at the + # revision the daily build would have used. Always the Daily entry -- + # a release is driven by release.yml, which passes its own revision. id: revision shell: bash env: diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index e6dcf24d..e3e2ca1a 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -8,6 +8,12 @@ name: build-kernel-debian # reaches this file has already decided the build belongs on the Debusine # path, so every job in it runs. # +# workflow_call only. One run of this workflow is one matrix entry, and a +# reusable workflow cannot fan itself out over several, so manual builds are +# dispatched from daily.yml -- which resolves the entries and calls this once +# per Debian entry it selected. That also keeps every value describing a build +# in ci/build-matrix.yaml, rather than in a dispatch form that drifts from it. +# # release-kernel-debian.yml is the same build with the release tail: it # promotes to a target Debusine workspace instead of publishing to S3. # @@ -99,90 +105,6 @@ on: description: 'Debusine token for the CI child workspace' required: false - # Ad-hoc build of one Debian suite. Release promotion is not offered here: - # a manual build never promotes, so it always takes the S3 publish path. - workflow_dispatch: - inputs: - build: - description: 'Name of this build, as listed in ci/build-matrix.yaml' - required: true - default: 'qcom-next-trixie' - type: string - flavour: - description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' - required: true - default: 'qcom-next' - type: string - suite: - description: 'Target Debian suite (trixie, forky, sid). An Ubuntu suite here is rejected before anything is built; dispatch build-kernel-ubuntu for those.' - required: true - default: 'trixie' - type: string - ref-strategy: - description: 'Kernel ref strategy' - required: true - default: 'latest_tag' - type: choice - options: - - latest_tag - - branch_tip - - pinned_ref - kernel-branch: - description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' - required: false - default: 'qcom-next' - tag-pattern: - description: 'Tag glob used only by latest_tag (ignored for branch_tip and pinned_ref)' - required: false - default: 'qcom-next-*' - git-clone: - description: 'Advanced: custom kernel repository URL' - required: false - default: 'https://github.com/qualcomm-linux/kernel' - srcpkg: - description: 'Advanced: Debian source package name' - required: false - default: 'linux-qcom-next' - binpkg: - description: 'Advanced: kernel image metapackage name' - required: false - default: 'linux-image-qcom-next' - kernel-config: - description: 'Advanced: extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' - required: false - default: '' - dkms: - description: 'Advanced: out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' - required: false - default: '' - debian-revision: - description: 'Advanced: Debian revision override (empty takes the Daily revision the matrix gives this build and suite)' - required: false - default: '' - localversion: - description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' - required: false - default: '' - kver-extra: - description: 'Advanced: extra package-version suffix (e.g. -ci42)' - required: false - default: '' - debug-build: - description: 'Advanced: enable the debug build configuration' - type: boolean - required: false - default: false - qcom-next-pr: - description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' - type: string - required: false - default: '' - kernel-topics-pr: - description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' - type: string - required: false - default: '' - permissions: contents: read packages: read diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 270c15b3..33a4b49f 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -12,6 +12,12 @@ name: build-kernel-ubuntu # entry for an Ubuntu suite is rejected by resolve-matrix.py rather than built # here and quietly not promoted. # +# workflow_call only. One run of this workflow is one matrix entry, and a +# reusable workflow cannot fan itself out over several, so manual builds are +# dispatched from daily.yml -- which resolves the entries and calls this once +# per Ubuntu entry it selected. That also keeps every value describing a build +# in ci/build-matrix.yaml, rather than in a dispatch form that drifts from it. +# # Jobs: # prepare (self-hosted): clone, patch, inject debian/, generate the source. # build (self-hosted): build the .deb files and upload them to S3. @@ -88,89 +94,6 @@ on: type: string default: '' - # Ad-hoc build of one Ubuntu suite. - workflow_dispatch: - inputs: - build: - description: 'Name of this build, as listed in ci/build-matrix.yaml' - required: true - default: 'qcom-next-resolute' - type: string - flavour: - description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' - required: true - default: 'qcom-next' - type: string - suite: - description: 'Target Ubuntu suite (noble, resolute). A Debian suite here is rejected before anything is built; dispatch build-kernel-debian for those.' - required: true - default: 'resolute' - type: string - ref-strategy: - description: 'Kernel ref strategy' - required: true - default: 'latest_tag' - type: choice - options: - - latest_tag - - branch_tip - - pinned_ref - kernel-branch: - description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' - required: false - default: 'qcom-next' - tag-pattern: - description: 'Tag glob used only by latest_tag (ignored for branch_tip and pinned_ref)' - required: false - default: 'qcom-next-*' - git-clone: - description: 'Advanced: custom kernel repository URL' - required: false - default: 'https://github.com/qualcomm-linux/kernel' - srcpkg: - description: 'Advanced: Debian source package name' - required: false - default: 'linux-qcom-next' - binpkg: - description: 'Advanced: kernel image metapackage name' - required: false - default: 'linux-image-qcom-next' - kernel-config: - description: 'Advanced: extra config fragments applied on top of all of debian/config-available/ (e.g. intree:arch/arm64/configs/qcom_debug.config)' - required: false - default: '' - dkms: - description: 'Advanced: out-of-tree DKMS modules to build and bundle into the kernel image, comma-separated and without the -dkms suffix (e.g. kgsl,camx); empty bundles nothing' - required: false - default: '' - debian-revision: - description: 'Advanced: Debian revision override (empty takes the Daily revision the matrix gives this build and suite)' - required: false - default: '' - localversion: - description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' - required: false - default: '' - kver-extra: - description: 'Advanced: extra package-version suffix (e.g. -ci42)' - required: false - default: '' - debug-build: - description: 'Advanced: enable the debug build configuration' - type: boolean - required: false - default: false - qcom-next-pr: - description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' - type: string - required: false - default: '' - kernel-topics-pr: - description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' - type: string - required: false - default: '' - permissions: contents: read packages: read diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 12eda419..6c06dd8c 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -1,9 +1,12 @@ name: daily # Daily kernel package builds. Runs at 3:00 PM PST (23:00 UTC), one parallel -# build per Daily entry in ci/build-matrix.yaml. Manual dispatch can run the -# full Daily matrix, a named set of builds, or every suite of one flavour. -# Build names and flavours are matrix values, so new ones need no workflow edit. +# build per Daily entry in ci/build-matrix.yaml. +# +# This is also the manual build entry point. Everything about a build comes +# from its matrix entry, so a dispatch only says which entries to run: "all", +# or a comma-separated list of build names. Build names are matrix values, so +# a new one needs no workflow edit. # # Matrix schema: ci/build-matrix.yaml # One entry is one generated package, so ci/scripts/resolve-matrix.py selects @@ -22,24 +25,29 @@ on: - cron: '0 23 * * *' workflow_dispatch: inputs: - build-scope: - description: 'Choose the whole matrix, named build(s), or every suite of one flavour' - type: choice - required: true - default: 'Selected builds' - options: - - 'Full matrix' - - 'Selected builds' - - 'Selected flavour (all suites)' builds: - description: 'Comma-separated build names, e.g. qcom-next-trixie,qcom-next-debug-forky (Selected builds)' + description: 'Builds to run: all, or comma-separated Daily build names from ci/build-matrix.yaml (e.g. qcom-next-trixie,qcom-next-debug-forky)' required: true - default: 'qcom-next-trixie' + default: 'all' type: string - flavour: - description: 'Kernel flavour to build in every suite it targets (Selected flavour)' - required: true - default: 'qcom-next' + # The matrix says nothing about these: they are one-off validation + # overrides, so they are inputs rather than entry fields. They apply to + # every selected build, and a scheduled run leaves them at their + # defaults. + debug-build: + description: 'Advanced: enable the debug build configuration' + required: false + default: false + type: boolean + qcom-next-pr: + description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' + required: false + default: '' + type: string + kernel-topics-pr: + description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' + required: false + default: '' type: string permissions: @@ -51,9 +59,9 @@ jobs: # configure-matrix: select the Daily entries from ci/build-matrix.yaml. # # Delegates to ci/scripts/resolve-matrix.py, which validates the whole matrix - # and returns the entries with type: Daily. Manual dispatch selects the - # complete matrix, a comma-separated list of build names, or one flavour - # across every suite it targets. + # and returns the entries with type: Daily. A dispatch narrows that to a + # comma-separated list of build names; "all", and a scheduled run, take the + # whole Daily matrix. # --------------------------------------------------------------------------- configure-matrix: name: Configure build matrix @@ -71,37 +79,24 @@ jobs: - name: Generate build matrix id: set-matrix env: - BUILD_SCOPE_INPUT: ${{ github.event.inputs.build-scope }} - INPUT_BUILDS: ${{ github.event.inputs.builds }} - INPUT_FLAVOUR: ${{ github.event.inputs.flavour }} + INPUT_BUILDS: ${{ inputs.builds }} run: | set -euo pipefail - BUILD_SCOPE="$BUILD_SCOPE_INPUT" + # A scheduled run carries no inputs, so it lands on the dispatch + # default: the whole Daily matrix. + BUILDS="${INPUT_BUILDS:-all}" - if [[ "${{ github.event_name }}" == "schedule" ]]; then - BUILD_SCOPE="Full matrix" + args=(--type Daily) + if [[ "$BUILDS" == "all" ]]; then + echo "Source: ci/build-matrix.yaml (every Daily entry)" + else + echo "Source: ci/build-matrix.yaml (builds: $BUILDS)" + # A name matching no Daily entry fails here, so a typo stops the + # run rather than quietly narrowing it to nothing. + args+=(--build "$BUILDS") fi - case "$BUILD_SCOPE" in - "Full matrix") - echo "Source: ci/build-matrix.yaml (full Daily matrix)" - args=(--type Daily) - ;; - "Selected builds") - echo "Source: ci/build-matrix.yaml (builds: $INPUT_BUILDS)" - args=(--type Daily --build "$INPUT_BUILDS") - ;; - "Selected flavour (all suites)") - echo "Source: ci/build-matrix.yaml (flavour: $INPUT_FLAVOUR, all suites)" - args=(--type Daily --flavour "$INPUT_FLAVOUR") - ;; - *) - echo "ERROR: unsupported build scope: $BUILD_SCOPE" >&2 - exit 1 - ;; - esac - # One selection, split by build path. The two families are built by # different workflows, so each gets the entries it builds and neither # run starts a job for the other. A narrowed dispatch can leave one @@ -159,6 +154,11 @@ jobs: localversion: ${{ matrix.localversion || '' }} kver-extra: ${{ matrix.kver_extra || '' }} debusine-parent-workspace: ${{ matrix.debusine_parent_workspace || '' }} + # Dispatch-only validation overrides. A scheduled run carries no inputs, + # so each falls back to the off value and builds the matrix as written. + debug-build: ${{ inputs.debug-build || false }} + qcom-next-pr: ${{ inputs.qcom-next-pr || '' }} + kernel-topics-pr: ${{ inputs.kernel-topics-pr || '' }} secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} @@ -190,3 +190,6 @@ jobs: debian-revision: ${{ matrix.debian_revision }} localversion: ${{ matrix.localversion || '' }} kver-extra: ${{ matrix.kver_extra || '' }} + debug-build: ${{ inputs.debug-build || false }} + qcom-next-pr: ${{ inputs.qcom-next-pr || '' }} + kernel-topics-pr: ${{ inputs.kernel-topics-pr || '' }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a96f4caa..b60b2d09 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,10 @@ name: release # packages are promoted from the ephemeral Debusine CI workspace to the stable # target workspace (e.g. qli) via debusine-action lib/release. # +# A dispatch says only which entries to release: "all", or a comma-separated +# list of build names. Everything else about a release comes from the entry, +# so there is nothing here to type that the matrix has not already reviewed. +# # To update a release ref: open a PR that changes branch_or_tag on every # Release entry for that flavour in ci/build-matrix.yaml. A flavour's Release # entries must agree on the ref, so resolve-matrix.py rejects the matrix if @@ -18,23 +22,10 @@ name: release on: workflow_dispatch: inputs: - release-scope: - description: 'Choose every suite of one flavour, or named build(s)' - type: choice - required: true - default: 'Selected flavour (all suites)' - options: - - 'Selected flavour (all suites)' - - 'Selected builds' - flavour: - description: 'Kernel flavour to release in every suite it targets (Selected flavour)' - required: true - default: 'qcom-next' - type: string builds: - description: 'Comma-separated build names, e.g. qcom-next-trixie,qcom-next-forky (Selected builds)' + description: 'Builds to release: all, or comma-separated Release build names from ci/build-matrix.yaml (e.g. qcom-next-trixie,qcom-next-forky)' required: true - default: 'qcom-next-trixie' + default: 'all' type: string permissions: @@ -60,28 +51,21 @@ jobs: - name: Generate release matrix id: set-matrix env: - RELEASE_SCOPE_INPUT: ${{ github.event.inputs.release-scope }} - INPUT_FLAVOUR: ${{ github.event.inputs.flavour }} - INPUT_BUILDS: ${{ github.event.inputs.builds }} + INPUT_BUILDS: ${{ inputs.builds }} run: | set -euo pipefail - RELEASE_SCOPE="$RELEASE_SCOPE_INPUT" + BUILDS="${INPUT_BUILDS:-all}" - case "$RELEASE_SCOPE" in - "Selected flavour (all suites)") - echo "Source: ci/build-matrix.yaml (flavour: $INPUT_FLAVOUR, all suites)" - args=(--type Release --flavour "$INPUT_FLAVOUR") - ;; - "Selected builds") - echo "Source: ci/build-matrix.yaml (builds: $INPUT_BUILDS)" - args=(--type Release --build "$INPUT_BUILDS") - ;; - *) - echo "ERROR: unsupported release scope: $RELEASE_SCOPE" >&2 - exit 1 - ;; - esac + args=(--type Release) + if [[ "$BUILDS" == "all" ]]; then + echo "Source: ci/build-matrix.yaml (every Release entry)" + else + echo "Source: ci/build-matrix.yaml (builds: $BUILDS)" + # A name matching no Release entry fails here, so a typo stops the + # run rather than quietly releasing a smaller set than intended. + args+=(--build "$BUILDS") + fi MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}") echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" diff --git a/README.md b/README.md index 3fefeeb2..00117e0a 100644 --- a/README.md +++ b/README.md @@ -127,10 +127,10 @@ variant is a matrix change, not a workflow redesign. | Workflow | Purpose | Trigger | | --- | --- | --- | -| `daily.yml` | Resolves and runs the Daily matrix. | Scheduled daily at `23:00 UTC`, or manual dispatch. | +| `daily.yml` | Resolves and runs the Daily matrix. The manual build entry point. | Scheduled daily at `23:00 UTC`, or manual dispatch. | | `release.yml` | Resolves and runs the Release matrix. | Manual dispatch only. | -| `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine and publishes it to S3. | Manual dispatch or called by Daily and PR build. | -| `build-kernel-ubuntu.yml` | Builds one Ubuntu-suite entry on the Docker path and publishes it to S3. | Manual dispatch or called by Daily and PR build. | +| `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine and publishes it to S3. | Called by Daily and PR build. | +| `build-kernel-ubuntu.yml` | Builds one Ubuntu-suite entry on the Docker path and publishes it to S3. | Called by Daily and PR build. | | `release-kernel-debian.yml` | Builds one Debian-suite entry in Debusine and promotes it to the release workspace. | Called by Release. | The three build workflows share their steps through two composite actions @@ -153,12 +153,12 @@ not take. Daily is the recurring build and artifact-publication path. - The scheduled run resolves the full `Daily` matrix. -- A manual run selects one **Build scope**: - - **Full matrix** builds every configured variant and suite. - - **Selected builds** builds a comma-separated list of build names, e.g. - `qcom-next-trixie,qcom-next-debug-forky`. - - **Selected flavour (all suites)** builds one flavour in every suite it - targets. +- A manual run says which entries to build in one **Builds** field: + - `all`, the default, builds every configured variant and suite. + - A comma-separated list of build names builds those entries, e.g. + `qcom-next-trixie,qcom-next-debug-forky`. A name matching no `Daily` entry + fails the run rather than narrowing it, and both families are selected from + one list, so a mixed list starts Debian and Ubuntu legs from one dispatch. - `latest_tag` resolves the newest matching dated tag; `branch_tip` resolves the configured branch directly. - Debian suites build in Debusine, then their `.deb` outputs are downloaded and @@ -170,11 +170,11 @@ Daily is the recurring build and artifact-publication path. Release is the controlled promotion path. -- It is manual only and uses one **Release scope** for a kernel variant: - - **Selected flavour (all suites)** is the normal release action and promotes - every configured Release suite for that flavour. - - **Selected builds** promotes a comma-separated list of build names when a - targeted action is required. +- It is manual only, and says which entries to release in one **Builds** field, + the same way Daily does: + - `all`, the default, promotes every configured `Release` entry. + - A comma-separated list of build names promotes those entries when a + targeted action is required, e.g. `qcom-next-trixie,qcom-next-forky`. - It uses the pinned `branch_or_tag` from the selected `Release` matrix row; it never resolves a newest tag. - Debian source and binary artifacts are built in per-flavour, per-suite @@ -186,11 +186,10 @@ Release is the controlled promotion path. the release credential and enforces the required approval gate before promotion to `qli`. -Direct `build-kernel-debian.yml` and `build-kernel-ubuntu.yml` dispatches are -build-only: neither has a promotion path to offer. Release promotion is -initiated exclusively by `release.yml`, which owns the target workspace and -production release controls, and is the only caller of -`release-kernel-debian.yml`. +A `daily.yml` dispatch is build-only: it publishes to the daily S3 path and has +no promotion path to offer. Release promotion is initiated exclusively by +`release.yml`, which owns the target workspace and production release controls, +and is the only caller of `release-kernel-debian.yml`. Only the Debian family has a release path at all, because promotion runs through Debusine. `resolve-matrix.py` rejects a `Release` entry for any other @@ -206,7 +205,7 @@ them to the workflow matrix as they stand. Each entry carries: | Field | Purpose | | --- | --- | -| `name` | The name of this one build, and nothing else: its Actions job name, and what a manual dispatch asks for. Unique within a delivery type, so a Daily and its Release may share a name. Never reaches a package name, a version, or a published path. Lowercase letters, digits, and internal hyphens only. | +| `name` | The name of this one build, and nothing else: its Actions job name, and what a manual dispatch asks for. Unique within a delivery type, so a Daily and its Release may share a name. Never reaches a package name, a version, or a published path. Lowercase letters, digits, and internal hyphens only, and not `all`, which a dispatch reads as every entry. | | `type` | `Daily` or `Release`. | | `suite` | The one suite this entry builds for. | | `flavour` | The kernel's own identity, and the only matrix field that reaches the built kernel. `derive-localversion.sh` makes it the LOCALVERSION suffix, so two flavours built from one ref get distinct kernel releases and their `linux-image` packages coexist. All entries sharing a flavour build the same package for different suites. | @@ -257,11 +256,11 @@ face value: - Where a suite has both, its `Daily` revision is its `Release` revision plus a trailing `~`. -A direct dispatch of a build workflow has no matrix context of its own, so when -its `debian-revision` input is empty the `prepare-kernel-source` action looks up -the `Daily` entry for the build name and suite it was given (`resolve-matrix.py ---field debian_revision`) and builds at the revision the daily build would have -used. +The build workflows pass their entry's own `debian_revision` through. A caller +with no entry in hand can leave the `debian-revision` input empty, and the +`prepare-kernel-source` action looks up the `Daily` entry for the build name and +suite it was given (`resolve-matrix.py --field debian_revision`) and builds at +the revision the daily build would have used. Each entry has a distinct prepared-source artifact, Debusine child workspace, and S3 path keyed by `flavour + suite`. This prevents two flavours that both @@ -326,10 +325,9 @@ flowchart LR flowchart TD subgraph triggers[Triggers] A1["daily.yml\nScheduled full matrix"] - A2["daily.yml\nManual full or filtered builds"] + A2["daily.yml\nManual: all or named builds"] A5["pr-build.yml\nFull Daily matrix on every PR"] - A3["release.yml\nManual full or filtered builds"] - A4["build-kernel-debian.yml\nbuild-kernel-ubuntu.yml\nManual one-off build"] + A3["release.yml\nManual: all or named builds"] end subgraph matrix[Matrix entry points] @@ -362,7 +360,6 @@ flowchart TD B2 --> C2 B5 --> C2 B4 --> C2 - A4 --> C2 C2 --> C3 & C4 C3 --> C5 & C6 C4 --> D1 @@ -492,42 +489,40 @@ out-of-tree module builds are required. ## Manual Builds -Use **Actions** → **build-kernel-debian** or **build-kernel-ubuntu** for a -one-off build, picking the one that builds the suite you want. These are -explicit override workflows, not matrix-derived delivery flows: use `daily.yml` -and `release.yml` for normal Daily and Release operations. Neither promotes. +Use **Actions** → **daily** → **Run workflow** for a one-off build, and +**Actions** → **release** to promote. Both are dispatched the same way: one +**Builds** field naming what to run. -`build`, `suite`, and `ref-strategy` are the required build selection. -All remaining package, configuration, and PR inputs are advanced overrides for -validation or debugging. Build name and suite are free-text matrix values -rather than static dropdowns, so adding a matrix entry never requires editing -the workflow UI. A suite belonging to the other family is rejected by the -`prepare` job before anything is cloned or built, so the only cost of picking -the wrong workflow is a fast failure. - -The available inputs are: +| Input | Default | Purpose | +| --- | --- | --- | +| `builds` | `all` | `all` runs every entry of that workflow's delivery type. Otherwise a comma-separated list of build `name` values from `ci/build-matrix.yaml`, e.g. `qcom-next-trixie,qcom-next-debug-forky`. | + +Everything else about a build — its suite, flavour, kernel repository and ref, +package names, config fragments, DKMS modules and Debian revision — comes from +the entry, so there is nothing to retype and nothing to get wrong. Build names +are free-text matrix values rather than a static dropdown, so adding a matrix +entry never requires editing the workflow UI, and a name that matches no entry +of the delivery type fails the run with the list of names that do. `daily` +routes each selected entry to the workflow that builds its family, so one +dispatch can name Debian and Ubuntu builds together. + +`daily` carries three further inputs, which the matrix deliberately says +nothing about because they belong to a one-off validation run rather than to a +delivery target. They apply to every selected build, and a scheduled run leaves +them at their defaults: | Input | Default | Purpose | | --- | --- | --- | -| `build` | `qcom-next-trixie` | Names this build; also selects the matrix entry a blank `debian-revision` is taken from. | -| `flavour` | `qcom-next` | Kernel flavour: the LOCALVERSION suffix, and so the kernel release identity. | -| `suite` | `trixie` | Target suite. | -| `ref-strategy` | `latest_tag` | `latest_tag`, `branch_tip`, or `pinned_ref`. | -| `kernel-branch` | `qcom-next` | Branch for `branch_tip`, or immutable ref for `pinned_ref`; ignored by `latest_tag`. | -| `tag-pattern` | `qcom-next-*` | Tag glob for `latest_tag`; ignored by `branch_tip` and `pinned_ref`. | -| `kernel-url` | `qualcomm-linux/kernel` | Advanced alternate kernel repository. | -| `srcpkg` | `linux-qcom-next` | Advanced source package identity override. | -| `binpkg` | `linux-image-qcom-next` | Advanced image metapackage identity override. | -| `kernel-config` | Empty | Advanced extra fragments applied on top of all of `debian/config-available/`, e.g. `intree:arch/arm64/configs/qcom_debug.config`. | -| `dkms` | Empty | Advanced comma-separated DKMS modules bundled into the image package, each without the `-dkms` suffix. Empty bundles none. | -| `debian-revision` | The matrix Daily revision | Advanced Debian revision override. Left empty, the build takes the `debian_revision` of the matrix's `Daily` entry for the selected variant and suite; direct builds always use the Daily entry since they are build-only and non-promoting. | -| `localversion` | Auto-derived | Advanced explicit `LOCALVERSION` override. | -| `kver-extra` | Empty | Advanced kernel-release suffix. | -| `debug-build` | `false` | Advanced debug configuration toggle. | - -The workflow also supports advanced Qualcomm-only PR overrides for validation -builds. Direct builds are artifact builds; Release promotion is performed only -through `release.yml`. +| `debug-build` | `false` | Advanced debug configuration toggle. For a lasting debug kernel, use the `qcom-next-debug` flavour instead. | +| `qcom-next-pr` | Empty | Advanced Qualcomm-only override: `qcom-next` PR numbers to merge before building. | +| `kernel-topics-pr` | Empty | Advanced Qualcomm-only override: `kernel-topics` PR numbers to apply as patches. | + +A `daily` dispatch is an artifact build and publishes to the daily S3 path; +Release promotion is performed only through `release.yml`. The build workflows +themselves (`build-kernel-debian.yml`, `build-kernel-ubuntu.yml`, +`release-kernel-debian.yml`) are `workflow_call` only and cannot be dispatched: +one run of each is one matrix entry, and a reusable workflow cannot fan itself +out over a list. ## Configuration @@ -586,8 +581,9 @@ To add a new suite (for an existing or new variant): [Overview](#overview)), and so it does not collide with another entry building the same `srcpkg`. -No workflow dispatch choices need to be updated: manual Daily and Release -inputs accept matrix-defined variant and suite strings. +No workflow dispatch choices need to be updated: the Daily and Release +dispatches take build names as free text, so a new entry is dispatchable by +name, and is picked up by `all`, as soon as it is merged. Run `ci/scripts/resolve-matrix.py --type Daily` and `--type Release` locally to validate a matrix change before pushing it; both validate the whole document, diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index 6c23b613..784809ea 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -24,7 +24,8 @@ # A build's name is what CI calls it: the Actions job name, and what a manual # dispatch asks for by name. It never reaches a package name or version, and # never a published path. It must be unique within a delivery type, so a Daily -# and the Release that supersedes it share one. +# and the Release that supersedes it share one. It may not be "all", which the +# daily and release dispatch forms read as every entry of their type. # # dkms varies by suite because it depends on which -dkms packages the # target archive carries: the Debian suites bundle camx and iris-vpu, which diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index edadb6a9..7afa7d3f 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -159,6 +159,12 @@ def family_for(suite): NAME_RE = re.compile(r"^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$") +# The dispatch forms of daily.yml and release.yml take one builds field, where +# "all" means every entry of the delivery type and anything else is a list of +# names. A build actually called all would be unreachable through them, so the +# matrix may not define one. +RESERVED_NAMES = ("all",) + # A Debian revision: no hyphen (that would start a new revision component) and # none of the characters dpkg rejects in a version. REVISION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.+~]*$") @@ -270,6 +276,12 @@ def check_entry(entry, report): if isinstance(value, str) and not NAME_RE.match(value): report(f"{field} must use lowercase letters, digits, and internal hyphens") + if entry.get("name") in RESERVED_NAMES: + report( + f"name {entry['name']} is reserved by the build workflows' dispatch " + "form, where it selects every entry rather than one of them" + ) + check_kernel_config(entry, report) check_dkms(entry, report) From af31628a338df04aad663e29cbadb83ad0a4af50 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 35/71] ci: release by promoting the nightly build, not by rebuilding it A release was a second build. release.yml took a pinned tag, built it from scratch in a fresh CI workspace, and promoted that -- bits the daily build had already produced from the same tag and tested. Everything else followed from the duplicate build: a second set of matrix entries to describe it, a type field to tell the two sets apart, ref_strategy rules per type, a release ref pinned in a file so that releasing meant editing the matrix, and a revision convention whose only job was to keep the two builds' versions apart. What shipped was never quite the artifact that passed the daily. So the daily build now promotes. An entry naming a target_workspace is copied into it once its build succeeds, and release.yml promotes onward from there and builds nothing at all: it names a version that already exists and asks Debusine to publish it into the release workspace. lib/release was already built for this -- it promotes between workspaces given a source package and a version, and defaults its target to qli-staging. That collapses the matrix to one kind of entry, 15 to 11. type is gone, and with it the per-type ref and revision rules; target_workspace is now an optional field on any Debian entry rather than a Release-only one, and the entries that name none -- mainline, next, the topic branch -- are built for early warning and published to S3 exactly as before. Releasing changes no file here: there is no ref to pin, because the version being released has already been built. The trailing ~ therefore stays on the version through release. Promoting the tested artifact means the archive gets the version it was built with, and a release that dropped the ~ would have to be a different build, which is the thing being removed. Nothing new upstream is now a green run rather than a failure. The version is a function of the resolved ref and the entry's revision, so a night on which the tracked tag has not moved rebuilds a version the archive already holds and Debusine rejects the duplicate. prepare reads the version out of the changelog it just generated and asks the target workspace whether it has it, and the build, the S3 publication and the promotion are skipped when it does. An unreachable workspace fails the run instead: a configuration problem must not look like a quiet night. Three kinds of run promote nowhere by construction. A PR build never forwards a target-workspace. A dispatch setting debug-build, qcom-next-pr or kernel-topics-pr builds something the matrix does not describe, so build-kernel-debian.yml declines to promote it -- guarded in the callee, where it holds for every caller rather than for the ones that remembered. And an entry naming no workspace has nowhere to go. release-kernel-debian.yml is deleted. It was the release tail of a build that no longer happens. Signed-off-by: Christopher Obbard --- .github/actions/debusine-build/action.yml | 16 +- .../actions/prepare-kernel-source/action.yml | 42 +- .github/workflows/build-kernel-debian.yml | 193 +++++++- .github/workflows/daily.yml | 49 ++- .github/workflows/pr-build.yml | 14 +- .github/workflows/release-kernel-debian.yml | 223 ---------- .github/workflows/release.yml | 169 ++++--- README.md | 416 +++++++++--------- ci/build-matrix.yaml | 149 ++----- ci/scripts/resolve-matrix.py | 205 +++------ 10 files changed, 696 insertions(+), 780 deletions(-) delete mode 100644 .github/workflows/release-kernel-debian.yml diff --git a/.github/actions/debusine-build/action.yml b/.github/actions/debusine-build/action.yml index e089dff9..17b3140a 100644 --- a/.github/actions/debusine-build/action.yml +++ b/.github/actions/debusine-build/action.yml @@ -3,11 +3,11 @@ description: > Generate the Debian source package from the prepared tree and submit it to Debusine, returning the CI workspace it was built in. -# Shared by the two Debian workflows, which differ only in what they do with -# the result: build-kernel-debian.yml downloads the .deb files and uploads them -# to S3, release-kernel-debian.yml promotes them to a target workspace. Holding -# these steps in a composite action lets each workflow declare its own runner, -# container and environment while keeping one copy of the build itself. +# Called by build-kernel-debian.yml, which decides what becomes of the result: +# every build is downloaded and uploaded to S3, and one whose entry names a +# target workspace is also promoted into it. Holding these steps in a composite +# action keeps the submission itself in one place, separate from the job that +# declares the runner, container and environment it happens in. # # The caller must have checked out pkg-linux-qcom into the workspace first, # which is also where this action comes from. Runs inside the @@ -41,9 +41,9 @@ inputs: extra-build-dep-workspaces: description: > Workspaces the build resolves its build-dependencies from, space - separated. A staging build may depend on a package version that has only - reached qli-staging; a release build reads qli alone, so a released - kernel is never built against something unreleased. + separated. The nightly build reads qli-staging as well as qli, because a + kernel may need a -dkms package version that has reached staging and not + yet been released. default: 'qli' debusine-action-ref: description: 'Ref of qualcomm-linux/debusine-action to use' diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index 9a993ad2..3811def2 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -20,6 +20,15 @@ description: > # kernel ref comes from a remote's tag list, and on pr-build.yml the inputs # come from the build matrix as the pull request wrote it. +outputs: + srcpkg_version: + description: > + Version of the source package this build will produce, read from the + generated debian/changelog. A caller that promotes the result into an + archive needs it before the build starts, to see whether that version is + already published there. + value: ${{ steps.version.outputs.srcpkg_version }} + inputs: build: description: 'Name of this build, as listed in ci/build-matrix.yaml' @@ -294,10 +303,9 @@ runs: - name: Resolve Debian revision # The build workflows pass the matrix entry's own debian-revision # straight through, so this lookup is the fallback for a caller that has - # no entry in hand and leaves the input empty: it looks up the Daily - # entry for the build name and suite it was given, and builds at the - # revision the daily build would have used. Always the Daily entry -- - # a release is driven by release.yml, which passes its own revision. + # no entry in hand and leaves the input empty: it looks up the entry for + # the build name and suite it was given, and builds at the revision the + # nightly build uses. id: revision shell: bash env: @@ -311,11 +319,10 @@ runs: echo "Using supplied Debian revision: $DEBIAN_REVISION" else DEBIAN_REVISION=$(ci/scripts/resolve-matrix.py \ - --type Daily \ --build "$BUILD_INPUT" \ --suite "$DISTRO" \ --field debian_revision) || { - echo "::error::No Daily matrix entry named $BUILD_INPUT for suite $DISTRO." + echo "::error::No matrix entry named $BUILD_INPUT for suite $DISTRO." echo "::error::Add one to ci/build-matrix.yaml, or set the debian-revision input to build this combination ad hoc." exit 1 } @@ -426,6 +433,29 @@ runs: "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" \ ./prepare-source.sh "${ARGS[@]}" + - name: Read the generated source version + # prepare-source.sh has just written debian/changelog, so the version + # this build will produce is now decided and can be reported to the + # caller. Taken from the changelog rather than recomputed here: that file + # is what the source package is built from, so nothing can disagree with + # it. Parsed with sed rather than dpkg-parsechangelog because these steps + # run on the runner host, which is not required to have dpkg-dev; the + # first line of a changelog is "srcpkg (version) suite; urgency=...". + id: version + shell: bash + env: + KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} + run: | + set -euo pipefail + SRCPKG_VERSION=$(sed -n '1s/^[^ ]* (\([^)]*\)).*/\1/p' \ + "$KERNEL_DIR/debian/changelog") + [[ -n "$SRCPKG_VERSION" ]] || { + echo "::error::Could not read a version from the generated debian/changelog." + exit 1 + } + echo "Source version: $SRCPKG_VERSION" + echo "srcpkg_version=$SRCPKG_VERSION" >> "$GITHUB_OUTPUT" + - name: Pack prepared source tree shell: bash env: diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index e3e2ca1a..0eac719a 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -1,12 +1,13 @@ name: build-kernel-debian -# Build a kernel package for a Debian-family suite and publish it to S3. +# Build a kernel package for a Debian-family suite, publish it to S3, and +# promote it into a Debusine workspace when the caller names one. # # Called per matrix entry by daily.yml and pr-build.yml, which select the # Debian entries with resolve-matrix.py --family debian and call only this # workflow for them. Nothing here is conditional on the suite: a caller that # reaches this file has already decided the build belongs on the Debusine -# path, so every job in it runs. +# path. # # workflow_call only. One run of this workflow is one matrix entry, and a # reusable workflow cannot fan itself out over several, so manual builds are @@ -14,13 +15,18 @@ name: build-kernel-debian # per Debian entry it selected. That also keeps every value describing a build # in ci/build-matrix.yaml, rather than in a dispatch form that drifts from it. # -# release-kernel-debian.yml is the same build with the release tail: it -# promotes to a target Debusine workspace instead of publishing to S3. +# There is no separate release build. An entry that names a target-workspace is +# promoted into it by this workflow, and release.yml promotes onward from that +# workspace without building anything, so what is released is the artifact this +# build produced and tested rather than a second build of the same ref. # # Jobs: -# prepare (self-hosted): clone, patch, inject debian/, generate the source. +# prepare (self-hosted): clone, patch, inject debian/, generate the source, +# and decide whether the version it produces is already published. # build (debusine-pkg-builder container): submit to Debusine. # publish (self-hosted): download the .deb files and upload them to S3. +# promote (debusine-pkg-builder container): copy the result into +# target-workspace. Skipped entirely when no workspace was named. on: workflow_call: @@ -89,6 +95,14 @@ on: description: 'Parent Debusine workspace for CI child workspace creation' type: string default: '' + target-workspace: + description: > + Debusine workspace to promote the built package into, empty to + promote nowhere. Set from the matrix entry's target_workspace by the + nightly build; left empty by pr-build.yml, whose builds must not + reach an archive. + type: string + default: '' qcom-next-pr: description: 'qcom-next PR numbers to merge (e.g. "42 43")' type: string @@ -111,7 +125,15 @@ permissions: jobs: # --------------------------------------------------------------------------- - # prepare: clone kernel source, inject packaging, run prepare-source.sh. + # prepare: clone kernel source, inject packaging, run prepare-source.sh, and + # report the version that produces. + # + # The version is decided here, by the ref this run resolved and the revision + # it was given, so this is the first point at which the run can tell whether + # there is anything new to build. When the caller named a target workspace + # and that version is already in it, every later job is skipped: the nightly + # build of a tag that has not moved would otherwise rebuild the same version + # and then fail promoting it as a duplicate. # --------------------------------------------------------------------------- prepare: name: Generate source package @@ -119,6 +141,9 @@ jobs: permissions: contents: read # actions/checkout packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder + outputs: + srcpkg_version: ${{ steps.prepare.outputs.srcpkg_version }} + already_published: ${{ steps.published.outputs.already_published }} steps: # debian/ and ci/ live on the same branch, so one checkout supplies both # the packaging and the scripts, the delivery matrix they read, and the @@ -132,6 +157,7 @@ jobs: persist-credentials: false - name: Prepare kernel source + id: prepare uses: ./.github/actions/prepare-kernel-source with: build: ${{ inputs.build }} @@ -153,12 +179,91 @@ jobs: qcom-next-pr: ${{ inputs.qcom-next-pr }} kernel-topics-pr: ${{ inputs.kernel-topics-pr }} + # Only needed to ask the target workspace what it already has, so it is + # checked out only when there is a target workspace to ask. + - name: Checkout debusine-action helpers + if: ${{ inputs.target-workspace != '' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: main + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + # The archive is the authority on whether there is anything to do. A + # rerun, a retry, and a night on which the tracked tag did not move all + # generate the version that is already published, because the version is + # a function of the resolved ref and the revision -- so asking here is + # what turns "nothing new upstream" into a skipped run rather than a + # rebuild that fails promoting a duplicate. + # + # Two ways of not getting an answer, deliberately treated differently. + # A workspace whose signing key cannot be fetched is a configuration + # problem -- it does not exist, or this token cannot see it -- and fails + # the run, because every later night would fail the same way and silence + # would hide it. An index that cannot be read once the workspace is + # reachable is reported as not having the version, which builds and lets + # the promotion fail loudly. Neither is allowed to look like a + # successful night on which nothing needed doing. + - name: Check the target workspace for this version + id: published + if: ${{ inputs.target-workspace != '' }} + env: + DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} + DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} + DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + DEBUSINE_WORKSPACE: ${{ inputs.target-workspace }} + SUITE: ${{ inputs.suite }} + SRCPKG: ${{ inputs.srcpkg }} + SRCPKG_VERSION: ${{ steps.prepare.outputs.srcpkg_version }} + run: | + # No -x: DEBUSINE_TOKEN is in env, keep xtrace off. + set -euo pipefail + debusine-action/lib/generate-apt-config || { + echo "::error::Cannot read $DEBUSINE_WORKSPACE. Check that the workspace exists and that DEBUSINE_TOKEN may read it." + exit 1 + } + sudo apt-get install -y --no-install-recommends devscripts + chdist create target-workspace + rm -f ~/.chdist/target-workspace/etc/apt/sources.list + install -d ~/.chdist/target-workspace/etc/apt/sources.list.d + install -d ~/.chdist/target-workspace/etc/apt/auth.conf.d + install -m 0644 debusine-ci.sources \ + ~/.chdist/target-workspace/etc/apt/sources.list.d/ + install -m 0600 debusine-ci-auth.conf \ + ~/.chdist/target-workspace/etc/apt/auth.conf.d/ + + if ! chdist apt-get target-workspace update; then + echo "::warning::Could not read $DEBUSINE_WORKSPACE; treating $SRCPKG $SRCPKG_VERSION as unpublished." + echo "already_published=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if chdist apt-cache target-workspace showsrc "$SRCPKG" 2>/dev/null \ + | grep -qxF "Version: $SRCPKG_VERSION"; then + echo "$SRCPKG $SRCPKG_VERSION is already in $DEBUSINE_WORKSPACE." + echo "Nothing new to build; the rest of this run is skipped." \ + >> "$GITHUB_STEP_SUMMARY" + echo "already_published=true" >> "$GITHUB_OUTPUT" + else + echo "$SRCPKG $SRCPKG_VERSION is not in $DEBUSINE_WORKSPACE yet." + echo "already_published=false" >> "$GITHUB_OUTPUT" + fi + # --------------------------------------------------------------------------- # build: generate the Debian source package and submit it to Debusine. + # + # Skipped, with everything after it, when prepare found this version already + # published in the target workspace. # --------------------------------------------------------------------------- build: name: Build package (Debusine) needs: prepare + if: ${{ needs.prepare.outputs.already_published != 'true' }} runs-on: ubuntu-latest permissions: contents: read # actions/checkout @@ -172,6 +277,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} outputs: workspace: ${{ steps.debusine.outputs.workspace }} + srcpkg_version: ${{ steps.debusine.outputs.srcpkg_version }} steps: # Only for the debusine-build action itself: the package being built # comes from the prepared-source artifact, not from this checkout. @@ -261,3 +367,78 @@ jobs: s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} path: deb-artifacts destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ + + # --------------------------------------------------------------------------- + # promote: copy the built package from the ephemeral CI workspace into the + # workspace the caller named, making it installable from that archive. + # + # This is what replaces a separate release build. The artifact promoted here + # is the one the build above produced, so an archive never carries a version + # that was not built and published by this run. + # + # The if: is the whole of the decision. A caller that named no workspace + # promotes nowhere, and a run carrying any of the validation overrides + # promotes nowhere either: those builds are deliberately not the kernel the + # matrix describes, and an archive must only ever receive that one. Guarding + # it here rather than in the caller means it holds for every caller, + # including a future one that forwards target-workspace without thinking + # about the overrides. + # --------------------------------------------------------------------------- + promote: + name: Promote to ${{ inputs.target-workspace }} + # build alone: it already needs prepare, so a run prepare skipped as + # already published skips the build and this job with it. + needs: build + if: >- + ${{ inputs.target-workspace != '' + && !inputs.debug-build + && inputs.qcom-next-pr == '' + && inputs.kernel-topics-pr == '' }} + runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout + packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder + environment: Staging + container: + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: main + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + - name: Promote packages to the target workspace + env: + DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} + DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + DEBUSINE_CI_WORKSPACE: ${{ needs.build.outputs.workspace }} + DEBUSINE_TARGET_WORKSPACE: ${{ inputs.target-workspace }} + SRCPKG_NAME: ${{ inputs.srcpkg }} + SRCPKG_VERSION: ${{ needs.build.outputs.srcpkg_version }} + SUITE: ${{ inputs.suite }} + run: | + set -eu + debusine-action/lib/release + + - name: Note the promoted version + env: + SRCPKG_NAME: ${{ inputs.srcpkg }} + SRCPKG_VERSION: ${{ needs.build.outputs.srcpkg_version }} + TARGET_WORKSPACE: ${{ inputs.target-workspace }} + run: | + echo "Promoted $SRCPKG_NAME $SRCPKG_VERSION to $TARGET_WORKSPACE" \ + >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 6c06dd8c..1272abb0 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -1,23 +1,29 @@ name: daily # Daily kernel package builds. Runs at 3:00 PM PST (23:00 UTC), one parallel -# build per Daily entry in ci/build-matrix.yaml. +# build per entry in ci/build-matrix.yaml. # # This is also the manual build entry point. Everything about a build comes # from its matrix entry, so a dispatch only says which entries to run: "all", # or a comma-separated list of build names. Build names are matrix values, so # a new one needs no workflow edit. # +# This is the only workflow that builds a kernel on a schedule, and every +# package the archives carry comes from a run of it: an entry naming a +# target_workspace is promoted into that workspace as part of its build. +# release.yml promotes onward from there and builds nothing, so nothing is +# ever released that this workflow has not built and tested. +# # Matrix schema: ci/build-matrix.yaml -# One entry is one generated package, so ci/scripts/resolve-matrix.py selects -# the entries with type: Daily and passes them through as they stand. Every -# field (name, suite, flavour, srcpkg, binpkg, kernel_config, dkms, -# debian_revision, git_clone, branch_or_tag, ref_strategy) becomes an input -# of the build workflow for the entry's family. name labels the job; flavour -# is what reaches the built kernel's version and its published paths. +# One entry is one generated package, so ci/scripts/resolve-matrix.py passes +# the selected entries through as they stand. Every field (name, suite, +# flavour, srcpkg, binpkg, kernel_config, dkms, debian_revision, git_clone, +# branch_or_tag, ref_strategy, target_workspace) becomes an input of the +# build workflow for the entry's family. name labels the job; flavour is what +# reaches the built kernel's version and its published paths. # -# To add a daily build target: add an entry to ci/build-matrix.yaml with -# type: Daily. No workflow changes needed. +# To add a daily build target: add an entry to ci/build-matrix.yaml. No +# workflow changes needed. on: schedule: @@ -26,7 +32,7 @@ on: workflow_dispatch: inputs: builds: - description: 'Builds to run: all, or comma-separated Daily build names from ci/build-matrix.yaml (e.g. qcom-next-trixie,qcom-next-debug-forky)' + description: 'Builds to run: all, or comma-separated build names from ci/build-matrix.yaml (e.g. qcom-next-trixie,qcom-next-debug-forky)' required: true default: 'all' type: string @@ -56,12 +62,11 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: select the Daily entries from ci/build-matrix.yaml. + # configure-matrix: select the entries to build from ci/build-matrix.yaml. # # Delegates to ci/scripts/resolve-matrix.py, which validates the whole matrix - # and returns the entries with type: Daily. A dispatch narrows that to a - # comma-separated list of build names; "all", and a scheduled run, take the - # whole Daily matrix. + # and returns its entries. A dispatch narrows that to a comma-separated list + # of build names; "all", and a scheduled run, take the whole matrix. # --------------------------------------------------------------------------- configure-matrix: name: Configure build matrix @@ -84,16 +89,16 @@ jobs: set -euo pipefail # A scheduled run carries no inputs, so it lands on the dispatch - # default: the whole Daily matrix. + # default: the whole matrix. BUILDS="${INPUT_BUILDS:-all}" - args=(--type Daily) + args=() if [[ "$BUILDS" == "all" ]]; then - echo "Source: ci/build-matrix.yaml (every Daily entry)" + echo "Source: ci/build-matrix.yaml (every entry)" else echo "Source: ci/build-matrix.yaml (builds: $BUILDS)" - # A name matching no Daily entry fails here, so a typo stops the - # run rather than quietly narrowing it to nothing. + # A name matching no entry fails here, so a typo stops the run + # rather than quietly narrowing it to nothing. args+=(--build "$BUILDS") fi @@ -154,6 +159,12 @@ jobs: localversion: ${{ matrix.localversion || '' }} kver-extra: ${{ matrix.kver_extra || '' }} debusine-parent-workspace: ${{ matrix.debusine_parent_workspace || '' }} + # An entry naming a workspace is promoted into it once its build + # succeeds; one that names none is built and published to S3 only. The + # entry is forwarded as written: build-kernel-debian.yml is what declines + # to promote a run carrying the validation overrides below, so a build + # that is not the plain nightly cannot reach an archive by any caller. + target-workspace: ${{ matrix.target_workspace || '' }} # Dispatch-only validation overrides. A scheduled run carries no inputs, # so each falls back to the off value and builds the matrix as written. debug-build: ${{ inputs.debug-build || false }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 8dfeea50..3ef53b0a 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -20,7 +20,7 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: select the Daily entries from the PR's build-matrix.yaml. + # configure-matrix: select the entries from the PR's build-matrix.yaml. # # The PR head, not the merge commit: a push mid-run must not change what this # run tested. A SHA also resolves for fork PRs, whose head branch does not @@ -48,7 +48,7 @@ jobs: # so a PR gets one job per leg it actually builds and no skipped job # for the path that leg does not take. for family in debian ubuntu; do - MATRIX=$(ci/scripts/resolve-matrix.py --type Daily \ + MATRIX=$(ci/scripts/resolve-matrix.py \ --family "$family" --allow-empty) echo "$family=$MATRIX" >> "$GITHUB_OUTPUT" echo "$family entries: $(echo "$MATRIX" | jq length)" @@ -59,14 +59,16 @@ jobs: # daily.yml. # # A PR gets one check per build leg and nothing else: neither workflow has a - # branch to skip, and a PR builds the whole Daily matrix, so both families - # have entries and neither guard fires. + # branch to skip, and a PR builds the whole matrix, so both families have + # entries and neither guard fires. # # fail-fast is disabled so a broken trixie leg still leaves forky and # resolute results on the PR. # - # Neither takes DEBUSINE_RELEASE_TOKEN: a PR build never promotes, so it - # calls the S3-publishing workflow and not release-kernel-debian.yml. + # target-workspace is deliberately not passed on. An entry carrying one is + # promoted into it by the nightly build; a PR build of the same entry only + # has to prove it builds, and must not put a pull request's kernel into an + # archive anyone installs from. # --------------------------------------------------------------------------- build-debian: name: ${{ matrix.name }} diff --git a/.github/workflows/release-kernel-debian.yml b/.github/workflows/release-kernel-debian.yml deleted file mode 100644 index f4eeb40a..00000000 --- a/.github/workflows/release-kernel-debian.yml +++ /dev/null @@ -1,223 +0,0 @@ -name: release-kernel-debian - -# Build a kernel package for a Debian-family suite and promote it to a release -# workspace in Debusine. -# -# Called per matrix entry by release.yml. Identical to build-kernel-debian.yml -# up to the Debusine build -- both share the prepare-kernel-source and -# debusine-build actions -- and differs only in the tail: this one promotes the -# built package to target-workspace rather than publishing it to the daily S3 -# path. Which tail a run wants is decided by the workflow that calls it, so -# neither file carries the other's tail as a job to be skipped. -# -# Only the Debian family has a release path: promotion runs through Debusine, -# and resolve-matrix.py rejects a Release entry for any other suite. -# -# Jobs: -# prepare (self-hosted): clone, patch, inject debian/, generate the source. -# build (debusine-pkg-builder container): submit to Debusine. -# release (debusine-pkg-builder container): promote to the target workspace. - -on: - workflow_call: - inputs: - build: - description: 'Name of this build, as listed in ci/build-matrix.yaml' - type: string - default: 'qcom-next-trixie' - flavour: - description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' - type: string - default: 'qcom-next' - suite: - description: 'Target Debian suite (trixie, forky, sid)' - type: string - default: 'trixie' - kernel-branch: - description: 'Immutable ref to build, from the entry branch_or_tag' - type: string - default: 'qcom-next' - git-clone: - description: 'Kernel repository URL' - type: string - default: 'https://github.com/qualcomm-linux/kernel' - ref-strategy: - description: 'Kernel ref strategy: a Release entry always pins one' - type: string - default: 'pinned_ref' - tag-pattern: - description: 'Tag glob, unused by pinned_ref and carried for symmetry with the Daily inputs' - type: string - default: '' - srcpkg: - description: 'Source package name' - type: string - default: 'linux-qcom-next' - binpkg: - description: 'Binary metapackage name' - type: string - default: 'linux-image-qcom-next' - kernel-config: - description: 'Extra config fragments applied on top of all of debian/config-available/' - type: string - default: '' - dkms: - description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' - type: string - default: '' - debian-revision: - description: 'Debian revision component of the package version' - type: string - default: '0qli' - localversion: - description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' - type: string - default: '' - kver-extra: - description: 'Extra suffix appended to the package version' - type: string - default: '' - debusine-parent-workspace: - description: 'Parent Debusine workspace for CI child workspace creation' - type: string - default: '' - target-workspace: - description: 'Debusine workspace to promote the built package into' - type: string - required: true - secrets: - DEBUSINE_USER: - description: 'Debusine account used to submit release builds' - required: false - DEBUSINE_TOKEN: - description: 'Debusine token for the CI child workspace' - required: false - DEBUSINE_RELEASE_TOKEN: - description: 'Debusine token for promotion to the release target workspace' - required: false - -permissions: - contents: read - packages: read - -jobs: - # --------------------------------------------------------------------------- - # prepare: clone kernel source, inject packaging, run prepare-source.sh. - # --------------------------------------------------------------------------- - prepare: - name: Generate source package - runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] - permissions: - contents: read # actions/checkout - packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder - steps: - - name: Checkout pkg-linux-qcom - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Prepare kernel source - uses: ./.github/actions/prepare-kernel-source - with: - build: ${{ inputs.build }} - suite: ${{ inputs.suite }} - expect-family: debian - flavour: ${{ inputs.flavour }} - kernel-url: ${{ inputs.git-clone }} - ref-strategy: ${{ inputs.ref-strategy }} - kernel-branch: ${{ inputs.kernel-branch }} - tag-pattern: ${{ inputs.tag-pattern }} - srcpkg: ${{ inputs.srcpkg }} - binpkg: ${{ inputs.binpkg }} - kernel-config: ${{ inputs.kernel-config }} - dkms: ${{ inputs.dkms }} - debian-revision: ${{ inputs.debian-revision }} - localversion: ${{ inputs.localversion }} - kver-extra: ${{ inputs.kver-extra }} - - # --------------------------------------------------------------------------- - # build: generate the Debian source package and submit it to Debusine. - # --------------------------------------------------------------------------- - build: - name: Build package (Debusine) - needs: prepare - runs-on: ubuntu-latest - permissions: - contents: read # actions/checkout - packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder - environment: Production - container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie - options: --user 0:0 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - outputs: - workspace: ${{ steps.debusine.outputs.workspace }} - srcpkg_version: ${{ steps.debusine.outputs.srcpkg_version }} - steps: - - name: Checkout pkg-linux-qcom - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Build in Debusine - id: debusine - uses: ./.github/actions/debusine-build - with: - suite: ${{ inputs.suite }} - flavour: ${{ inputs.flavour }} - debusine-host: ${{ vars.DEBUSINE_HOST }} - debusine-scope: ${{ vars.DEBUSINE_SCOPE }} - debusine-user: ${{ secrets.DEBUSINE_USER }} - debusine-token: ${{ secrets.DEBUSINE_TOKEN }} - debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} - # A release build reads qli alone, so a released kernel is never - # built against something that has not been released alongside it. - extra-build-dep-workspaces: qli - - # --------------------------------------------------------------------------- - # release: promote the built package from the CI workspace to the target. - # --------------------------------------------------------------------------- - release: - name: Release to Debusine - needs: build - runs-on: ubuntu-latest - permissions: - contents: read # actions/checkout - packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder - environment: Production - container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie - options: --user 0:0 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - defaults: - run: - shell: bash - steps: - - name: Checkout debusine-action helpers - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - repository: qualcomm-linux/debusine-action - ref: main - path: debusine-action - fetch-depth: 1 - sparse-checkout: | - lib - - - name: Promote packages to target workspace - env: - DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} - DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} - DEBUSINE_CI_WORKSPACE: ${{ needs.build.outputs.workspace }} - DEBUSINE_TARGET_WORKSPACE: ${{ inputs.target-workspace }} - SRCPKG_NAME: ${{ inputs.srcpkg }} - SRCPKG_VERSION: ${{ needs.build.outputs.srcpkg_version }} - SUITE: ${{ inputs.suite }} - run: | - set -ex - debusine-action/lib/release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b60b2d09..ad1cb17d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,32 +1,45 @@ name: release -# Release kernel package builds. Triggered manually only. +# Promote already-built kernel packages into the release workspace. # -# Reads the Release entries from ci/build-matrix.yaml and builds each one using -# the matrix-selected ref strategy and branch_or_tag. After a successful build, -# packages are promoted from the ephemeral Debusine CI workspace to the stable -# target workspace (e.g. qli) via debusine-action lib/release. +# This workflow builds nothing. Every package it can release was built, tested +# and published by daily.yml, which promotes each entry carrying a +# target_workspace into that workspace as part of the nightly run. A release +# copies one of those versions onward into the release workspace, so what ships +# is the artifact that was tested rather than a second build of the same ref. # -# A dispatch says only which entries to release: "all", or a comma-separated -# list of build names. Everything else about a release comes from the entry, -# so there is nothing here to type that the matrix has not already reviewed. +# That is why there is no release ref anywhere in ci/build-matrix.yaml, and why +# releasing needs no change to it: a release names a version that already +# exists, and the matrix describes what is built rather than what has been +# blessed. # -# To update a release ref: open a PR that changes branch_or_tag on every -# Release entry for that flavour in ci/build-matrix.yaml. A flavour's Release -# entries must agree on the ref, so resolve-matrix.py rejects the matrix if -# only some of them are updated. +# A dispatch says which builds to release and which version of them. Every +# selected entry promotes -: the +# upstream half is the kernel and the date it was cut, identical across the +# suites built from one ref, and the revision half is the entry's own. Check +# the version against the source workspace listing before dispatching -- a +# version that is not there fails the promotion rather than releasing +# something else. # -# This workflow intentionally has no schedule trigger. Releases are always -# initiated manually after validation. +# Triggered manually only, and never on a schedule: a release is a decision. on: workflow_dispatch: inputs: builds: - description: 'Builds to release: all, or comma-separated Release build names from ci/build-matrix.yaml (e.g. qcom-next-trixie,qcom-next-forky)' + description: 'Builds to release: all, or comma-separated build names from ci/build-matrix.yaml (e.g. qcom-next-trixie,qcom-next-forky)' required: true default: 'all' type: string + upstream-version: + description: 'Version to release, without the Debian revision (e.g. 7.2.0~rc7+20260821)' + required: true + type: string + release-workspace: + description: 'Debusine workspace to release into' + required: true + default: 'qli' + type: string permissions: contents: read @@ -34,7 +47,13 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: select the Release entries from ci/build-matrix.yaml. + # configure-matrix: select the entries that can be released. + # + # An entry is releasable when it names a target_workspace, because that is + # the workspace holding the nightly artifacts to promote from. The entries + # without one -- mainline, next, the topic branches -- are built for early + # warning and published to S3, so there is nothing to release and asking for + # one by name is a mistake worth reporting. # --------------------------------------------------------------------------- configure-matrix: name: Configure release matrix @@ -57,62 +76,100 @@ jobs: BUILDS="${INPUT_BUILDS:-all}" - args=(--type Release) + args=() if [[ "$BUILDS" == "all" ]]; then - echo "Source: ci/build-matrix.yaml (every Release entry)" + echo "Source: ci/build-matrix.yaml (every releasable entry)" else echo "Source: ci/build-matrix.yaml (builds: $BUILDS)" - # A name matching no Release entry fails here, so a typo stops the - # run rather than quietly releasing a smaller set than intended. + # A name matching no entry fails here, so a typo stops the run + # rather than quietly releasing a smaller set than intended. args+=(--build "$BUILDS") fi - MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}") + + # An entry with no target_workspace has no workspace to promote out + # of, so it is not releasable however it was selected. + releasable='[.[] | select(.target_workspace != "" and .target_workspace != null)]' + MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}" | jq -c "$releasable") + + if [[ "$(echo "$MATRIX" | jq length)" -eq 0 ]]; then + # Listed from the whole matrix, not from the failed selection, + # which by definition contains none of them. + ALL=$(ci/scripts/resolve-matrix.py \ + | jq -r "$releasable"' | map(.name) | join(", ")') + echo "::error::None of the selected builds names a target_workspace, so none of them has anything to release." + echo "::error::Releasable builds: $ALL" + exit 1 + fi echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" echo "Matrix entries: $(echo "$MATRIX" | jq length)" # --------------------------------------------------------------------------- - # build: one parallel job per Release matrix entry. - # - # Uses the ref strategy and branch_or_tag selected by the matrix. - # release-kernel-debian.yml builds exactly as the daily workflow does and - # then promotes to target_workspace, rather than publishing to the daily S3 - # path. Choosing the workflow here is what makes that the whole of the run: - # no job in it exists to publish the other way and be skipped. + # promote: one parallel job per entry, copying its version from the workspace + # the nightly build put it in into the release workspace. # - # There is no Ubuntu counterpart. Promotion runs through Debusine, which - # builds only the Debian family, so resolve-matrix.py rejects a Release entry - # for any other suite and this matrix is Debian by construction. + # Nothing is built, so there is no prepare job and no self-hosted runner: + # lib/release asks Debusine to publish an artifact it already holds. Both + # workspaces are Debusine workspaces, so this is the same operation the + # nightly build performs, one step further along. # --------------------------------------------------------------------------- - build: + promote: name: ${{ matrix.name }} needs: configure-matrix strategy: fail-fast: false matrix: include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} + runs-on: ubuntu-latest permissions: - contents: read # actions/checkout in the called workflow - packages: read # ghcr.io images it pulls - uses: ./.github/workflows/release-kernel-debian.yml - with: - suite: ${{ matrix.suite }} - build: ${{ matrix.name }} - flavour: ${{ matrix.flavour }} - git-clone: ${{ matrix.git_clone }} - kernel-branch: ${{ matrix.branch_or_tag }} - ref-strategy: ${{ matrix.ref_strategy }} - tag-pattern: ${{ matrix.tag_pattern || '' }} - srcpkg: ${{ matrix.srcpkg }} - binpkg: ${{ matrix.binpkg }} - kernel-config: ${{ matrix.kernel_config }} - dkms: ${{ matrix.dkms }} - debian-revision: ${{ matrix.debian_revision }} - localversion: ${{ matrix.localversion || '' }} - kver-extra: ${{ matrix.kver_extra || '' }} - debusine-parent-workspace: ${{ matrix.debusine_parent_workspace || '' }} - target-workspace: ${{ matrix.target_workspace }} - secrets: - DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} - DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} + contents: read # actions/checkout + packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder + # Production carries the release credential and the approval gate: a + # promotion into the release workspace is the one step here that a person + # has to agree to. + environment: Production + container: + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: main + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + - name: Promote to the release workspace + env: + DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} + DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} + # The entry's own target_workspace is the source: it is where the + # nightly build of this entry put the artifact being released. + DEBUSINE_CI_WORKSPACE: ${{ matrix.target_workspace }} + DEBUSINE_TARGET_WORKSPACE: ${{ inputs.release-workspace }} + SRCPKG_NAME: ${{ matrix.srcpkg }} + UPSTREAM_VERSION: ${{ inputs.upstream-version }} + DEBIAN_REVISION: ${{ matrix.debian_revision }} + SUITE: ${{ matrix.suite }} + run: | + set -eu + # The revision is the entry's, not the dispatcher's: a suite's + # packages are versioned to sort against the other suites, so the + # full version differs per entry even though the kernel does not. + export SRCPKG_VERSION="${UPSTREAM_VERSION}-${DEBIAN_REVISION}" + echo "Releasing $SRCPKG_NAME $SRCPKG_VERSION" \ + "from $DEBUSINE_CI_WORKSPACE to $DEBUSINE_TARGET_WORKSPACE" + debusine-action/lib/release + echo "Released $SRCPKG_NAME $SRCPKG_VERSION to $DEBUSINE_TARGET_WORKSPACE" \ + >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 00117e0a..ce384608 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,17 @@ takes its packaging and its CI scripts from the same commit. The CI model is matrix-driven. This single repository can deliver multiple kernel variants, each with independent source/package identity, kernel source and ref strategy, configuration fragments, Debian revision, target suites, and -release destination. +archive destination. Every entry in the matrix is one isolated build leg producing one package. A -kernel flavour owns as many entries as it has suites and delivery types. +kernel flavour owns as many entries as it has suites. ### Configured variants -| Variant | Source package | Image metapackage | Daily suites | Release suites | Notes | -|---------|----------------|-------------------|--------------|----------------|-------| -| `qcom-next` | `linux-qcom-next` | `linux-image-qcom-next` | trixie, forky, resolute | trixie, forky | Standard kernel | -| `qcom-next-debug` | `linux-qcom-next-debug` | `linux-image-qcom-next-debug` | trixie, forky | trixie, forky | Adds `arch/arm64/configs/qcom_debug.config` and `kernel/configs/debug.config` from the kernel source, via `intree:` entries | +| Variant | Source package | Image metapackage | Suites | Promoted to | Notes | +|---------|----------------|-------------------|--------|-------------|-------| +| `qcom-next` | `linux-qcom-next` | `linux-image-qcom-next` | trixie, forky, resolute | `qli-staging` (trixie, forky) | Standard kernel | +| `qcom-next-debug` | `linux-qcom-next-debug` | `linux-image-qcom-next-debug` | trixie, forky | `qli-staging` | Adds `arch/arm64/configs/qcom_debug.config` and `kernel/configs/debug.config` from the kernel source, via `intree:` entries | | `qcom-arduino` | `linux-qcom-arduino` | `linux-image-qcom-arduino` | trixie, forky | none | Arduino hardware-enablement topic branch (`early/hwe/arduino` of `kernel-topics`) | | `mainline` | `linux-mainline` | `linux-image-mainline` | trixie, forky | none | Tip of Linus's tree, tracked for early warning of upstream breakage. No DKMS modules | | `next` | `linux-next` | `linux-image-next` | trixie, forky | none | Newest `next-YYYYMMDD` tag of linux-next. No DKMS modules | @@ -35,29 +35,32 @@ image package that can be installed alongside the others. The flavour is what the kernel is; a build's `name` is only what CI calls it. See [docs/version.md](docs/version.md) for how the version strings are composed. -The last three track a moving upstream — a branch tip, or a tag cut every -night — and so have no `Release` entries: a release must name an immutable ref -to promote. They are built and published daily and never promoted to `qli`. +The last three name no `target_workspace`: they track a moving upstream for +early warning, are published to S3, and never reach an archive anyone installs +from. `ci/build-matrix.yaml` is the source of truth; this table is a summary. -Two entry points use the same reusable build pipeline: +There is one kind of entry, and one workflow that builds one: -- **Daily** uses the matrix-selected latest-tag or branch-tip strategy and - builds every configured Daily suite. -- **Release** uses a pinned matrix ref and promotes successful Debian packages - to the selected production Debusine workspace. +- **`daily.yml`** builds every entry nightly, using the matrix-selected + latest-tag or branch-tip strategy, and promotes each entry that names a + `target_workspace` into that workspace. +- **`release.yml`** builds nothing. It promotes a version that is already in a + staging workspace onward into the release workspace. -One entry in `builds` is one generated package: a single `name`, a single -`type`, and a single `suite`. The `name` labels that one build and nothing -else — it is the Actions job name and what a manual dispatch asks for. Nothing -is expanded or derived at resolve time, so what an entry says is what gets -built: +Releasing therefore changes no file in this repository. The matrix describes +what is built; which build has been blessed is a decision made at dispatch +time, against versions that already exist. + +One entry in `builds` is one generated package: a single `name` for a single +`suite`. The `name` labels that one build and nothing else — it is the Actions +job name and what a manual dispatch asks for. Nothing is expanded or derived at +resolve time, so what an entry says is what gets built: ```yaml builds: - name: qcom-next-trixie - type: Daily suite: trixie flavour: qcom-next git_clone: https://github.com/qualcomm-linux/kernel @@ -73,24 +76,7 @@ builds: - iris-vpu - audioreach debian_revision: '0qli1~bpo13+1~' - - - name: qcom-next-trixie - type: Release - suite: trixie - flavour: qcom-next - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: - ref_strategy: pinned_ref - srcpkg: linux-qcom-next - binpkg: linux-image-qcom-next - kernel_config: [] - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1~bpo13+1' - target_workspace: qli + target_workspace: qli-staging ``` Entries are written out in full rather than sharing YAML anchors, so each one @@ -100,25 +86,24 @@ consistency that duplication would otherwise put at risk — see Each entry states its `debian_revision` outright. The configured values are: -| Suite | Daily | Release | -| --- | --- | --- | -| Trixie | `0qli1~bpo13+1~` | `0qli1~bpo13+1` | -| Forky | `0qli1~` | `0qli1` | -| Resolute | `0qli1~26.04.1~` | (not a configured Release suite) | - -`~` always sorts below the same prefix without it in Debian version -ordering, so a suite's Daily always sorts below its Release; `resolve-matrix.py` -requires a Daily revision to be exactly its Release revision plus a trailing -`~`. Ordering across *different* suites depends entirely on the configured -revisions: with the values above, Resolute < Trixie < Forky for the same -delivery type, matching a Debian-backports-then-unstable promotion chain. -This is a deliberate ordering policy, not an automatic guarantee — adding a -suite means choosing a revision that sorts where that suite belongs relative to -the others. One nuance to be aware of: because Forky's revision carries no -suite component, its Daily revision ends immediately after the trailing `~`, so -Trixie Daily does not sort below Forky Daily even though Trixie Release sorts -below Forky Release. This does not affect the supported Release-to-Release -upgrade path. +| Suite | Revision | +| --- | --- | +| Trixie | `0qli~bpo13+1~` | +| Forky | `0qli~` | +| Resolute | `0qli~26.04.1~` | + +The trailing `~` marks the version as one nobody has blessed yet, and it stays +on the version through release. That is a consequence of releasing by +promotion: what reaches `qli` is the artifact `qli-staging` holds, byte for +byte, so its version is the version it was built with. A release that dropped +the `~` would have to be a different build, which is exactly what this design +removes. + +Ordering across *different* suites depends entirely on the configured +revisions: with the values above, Resolute < Trixie < Forky, matching a +Debian-backports-then-unstable promotion chain. This is a deliberate ordering +policy, not an automatic guarantee — adding a suite means choosing a revision +that sorts where that suite belongs relative to the others. `ci/build-matrix.yaml` is the authoritative configuration. Adding a kernel variant is a matrix change, not a workflow redesign. @@ -127,72 +112,86 @@ variant is a matrix change, not a workflow redesign. | Workflow | Purpose | Trigger | | --- | --- | --- | -| `daily.yml` | Resolves and runs the Daily matrix. The manual build entry point. | Scheduled daily at `23:00 UTC`, or manual dispatch. | -| `release.yml` | Resolves and runs the Release matrix. | Manual dispatch only. | -| `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine and publishes it to S3. | Called by Daily and PR build. | +| `daily.yml` | Resolves and runs the matrix. The manual build entry point. | Scheduled daily at `23:00 UTC`, or manual dispatch. | +| `release.yml` | Promotes an already-built version into the release workspace. Builds nothing. | Manual dispatch only. | +| `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine, publishes it to S3, and promotes it into the entry's `target_workspace` if it has one. | Called by Daily and PR build. | | `build-kernel-ubuntu.yml` | Builds one Ubuntu-suite entry on the Docker path and publishes it to S3. | Called by Daily and PR build. | -| `release-kernel-debian.yml` | Builds one Debian-suite entry in Debusine and promotes it to the release workspace. | Called by Release. | -The three build workflows share their steps through two composite actions -rather than through a common orchestrator workflow: +The two build workflows share their steps through two composite actions rather +than through a common orchestrator workflow: | Action | Used by | | --- | --- | -| `.github/actions/prepare-kernel-source` | All three, as the `prepare` job. | -| `.github/actions/debusine-build` | The two Debian workflows, as the `build` job. | +| `.github/actions/prepare-kernel-source` | Both, as the `prepare` job. | +| `.github/actions/debusine-build` | The Debian workflow, as the `build` job. | Which of them a build leg calls is decided by the caller, from the entry's suite: `resolve-matrix.py --family debian|ubuntu` splits the selection, and each family's entries call only the workflow that builds them. Nothing inside a -build workflow is conditional on the suite or on whether the run releases, so a -run starts exactly the jobs it needs and shows no skipped job for a path it did -not take. +build workflow is conditional on the suite, so a run starts exactly the jobs it +needs and shows no skipped job for a path it did not take. ### Daily -Daily is the recurring build and artifact-publication path. +Daily is the only workflow that builds a kernel, and every package in every +archive comes from a run of it. -- The scheduled run resolves the full `Daily` matrix. +- The scheduled run resolves the full matrix. - A manual run says which entries to build in one **Builds** field: - `all`, the default, builds every configured variant and suite. - A comma-separated list of build names builds those entries, e.g. - `qcom-next-trixie,qcom-next-debug-forky`. A name matching no `Daily` entry - fails the run rather than narrowing it, and both families are selected from - one list, so a mixed list starts Debian and Ubuntu legs from one dispatch. + `qcom-next-trixie,qcom-next-debug-forky`. A name matching no entry fails + the run rather than narrowing it, and both families are selected from one + list, so a mixed list starts Debian and Ubuntu legs from one dispatch. - `latest_tag` resolves the newest matching dated tag; `branch_tip` resolves the configured branch directly. - Debian suites build in Debusine, then their `.deb` outputs are downloaded and uploaded to the configured S3 bucket. +- An entry naming a `target_workspace` is then promoted into it with Debusine's + `package-publish` workflow, making it installable from that archive. - `resolute` stays on the Docker-based Ubuntu path and uploads its package outputs to the existing temporary-package S3 location. +Because the package version is a function of the resolved ref and the entry's +revision, a night on which the tracked tag has not moved would rebuild a +version the archive already has and then fail promoting the duplicate. The +`prepare` job therefore asks the target workspace whether it already holds the +version this run would produce, and skips the build, the S3 publication and the +promotion when it does. A run that skips this way is green: nothing was wrong, +there was simply nothing new upstream. + +Three kinds of run deliberately promote nowhere: a PR build, which never +forwards a `target_workspace`; a dispatch setting any of the advanced +validation overrides (`debug-build`, `qcom-next-pr`, `kernel-topics-pr`), +whose kernel is by definition not the one the matrix describes; and any entry +that names no workspace. + ### Release -Release is the controlled promotion path. - -- It is manual only, and says which entries to release in one **Builds** field, - the same way Daily does: - - `all`, the default, promotes every configured `Release` entry. - - A comma-separated list of build names promotes those entries when a - targeted action is required, e.g. `qcom-next-trixie,qcom-next-forky`. -- It uses the pinned `branch_or_tag` from the selected `Release` matrix row; it - never resolves a newest tag. -- Debian source and binary artifacts are built in per-flavour, per-suite - Debusine CI workspaces. -- Successful builds are promoted with Debusine's `package-publish` workflow to - the `qli` workspace, where they are available through the production Debusine - APT repository. -- The Release job runs in the **Production** GitHub environment. This provides - the release credential and enforces the required approval gate before - promotion to `qli`. - -A `daily.yml` dispatch is build-only: it publishes to the daily S3 path and has -no promotion path to offer. Release promotion is initiated exclusively by -`release.yml`, which owns the target workspace and production release controls, -and is the only caller of `release-kernel-debian.yml`. - -Only the Debian family has a release path at all, because promotion runs -through Debusine. `resolve-matrix.py` rejects a `Release` entry for any other +Release is the controlled promotion path, and it builds nothing at all. + +- It is manual only, and takes three fields: which **Builds** to release (`all` + or a comma-separated list, as Daily does), the **Upstream version** to + release, and the **Release workspace** to release into (`qli` by default). +- Each selected entry promotes `-` from + its own `target_workspace` into the release workspace. The upstream half is + the kernel and the date its tag was cut, and is the same across the suites + built from one ref; the revision half is the entry's own, because a suite's + packages are versioned to sort against the other suites. +- Only an entry that names a `target_workspace` can be released. There is + nowhere to promote from otherwise, so selecting only such entries fails the + run with the list of the ones that can be released. +- Promotion uses Debusine's `package-publish` workflow, the same operation the + nightly build performs one step earlier. +- The job runs in the **Production** GitHub environment, which provides the + release credential and enforces the approval gate. + +Releasing changes no file in this repository: there is no ref to pin and no +entry to add, because the version being released has already been built. The +release decision is the dispatch. + +Only the Debian family has a promotion path at all, because promotion runs +through Debusine. `resolve-matrix.py` rejects a `target_workspace` on any other suite rather than letting it build and then silently not promote. ## Matrix Model @@ -200,13 +199,12 @@ suite rather than letting it build and then silently not promote. `ci/build-matrix.yaml` is a mapping with exactly one top-level key, `builds`. One entry in it is one generated package, so there is no expansion step: `ci/scripts/resolve-matrix.py` validates the whole document, -selects the entries matching the requested type, variant, and suite, and hands +selects the entries matching the requested names, variant, and suite, and hands them to the workflow matrix as they stand. Each entry carries: | Field | Purpose | | --- | --- | -| `name` | The name of this one build, and nothing else: its Actions job name, and what a manual dispatch asks for. Unique within a delivery type, so a Daily and its Release may share a name. Never reaches a package name, a version, or a published path. Lowercase letters, digits, and internal hyphens only, and not `all`, which a dispatch reads as every entry. | -| `type` | `Daily` or `Release`. | +| `name` | The name of this one build, and nothing else: its Actions job name, and what a manual dispatch asks for. Unique across the matrix. Never reaches a package name, a version, or a published path. Lowercase letters, digits, and internal hyphens only, and not `all`, which a dispatch reads as every entry. | | `suite` | The one suite this entry builds for. | | `flavour` | The kernel's own identity, and the only matrix field that reaches the built kernel. `derive-localversion.sh` makes it the LOCALVERSION suffix, so two flavours built from one ref get distinct kernel releases and their `linux-image` packages coexist. All entries sharing a flavour build the same package for different suites. | | `git_clone` | Kernel source repository. | @@ -217,50 +215,41 @@ them to the workflow matrix as they stand. Each entry carries: | `binpkg` | Kernel image metapackage name. | | `kernel_config` | Extra fragments applied on top of `debian/config-available/`, all of which is applied to every build, one per list element. A bare name selects `debian/config-available/.config`; an `intree:` entry names a fragment shipped by the kernel source, as a path relative to the kernel source root (e.g. `intree:arch/arm64/configs/qcom_debug.config`), so it stays versioned with the kernel it targets. Empty for variants that need nothing beyond `config-available/`; today it carries only `intree:` fragments. `resolve-matrix.py` joins it into the comma-separated `kernel-config` workflow input. | | `dkms` | Out-of-tree DKMS modules built and bundled into the image package, one per list element, each named without the `-dkms` suffix (e.g. `kgsl`). Each needs a `-dkms` package in the suite being built for, so this varies between suites. An empty list bundles nothing. A listed module is a presence contract: a build fails rather than shipping an image without it. `resolve-matrix.py` joins it into the comma-separated `dkms` workflow input, which reaches `prepare-source.sh --dkms`; see [debian/README.md](debian/README.md) for what the packaging does with it. | -| `debian_revision` | The Debian revision this package is built at, stated outright. Daily revisions end in `~`; Release revisions do not. | +| `debian_revision` | The Debian revision this package is built at, stated outright. Carried into every archive the package reaches, because a release promotes the built artifact rather than rebuilding it. | | `localversion`, `kver_extra` | Optional version overrides forwarded to packaging. | | `debusine_parent_workspace` | Optional parent workspace override for the variant's CI child workspaces. | -| `target_workspace` | Debusine destination for Release entries only. | +| `target_workspace` | Debusine workspace the nightly build promotes this entry into, and the workspace `release.yml` later promotes it out of. Optional: an entry without one is built and published to S3 and reaches no archive. Debian suites only. | `resolve-matrix.py` rejects the matrix — before any build job starts — where an entry has an unknown field or a missing required one, a malformed variant or -suite identifier, a `ref_strategy` its `type` does not allow (`Daily` must -track something moving, `Release` must be pinned), a `tag_pattern` without -`latest_tag`, a `target_workspace` on a `Daily` entry or none on a `Release` -one, a `kernel_config` fragment that escapes the kernel source root or collides -with another fragment's filename, a `dkms` entry that is not a package name -stem or repeats, or a `debian_revision` that is not a valid Debian revision or -carries the wrong trailing `~` for its `type`. +suite identifier, an unknown `ref_strategy`, a `tag_pattern` without +`latest_tag`, a `target_workspace` on a non-Debian suite, a `kernel_config` +fragment that escapes the kernel source root or collides with another +fragment's filename, a `dkms` entry that is not a package name stem or repeats, +or a `debian_revision` that is not a valid Debian revision. Because entries are written out in full, the resolver also checks the invariants that span them, which is what makes the duplication safe to read at face value: -- No two entries of one `type` share a `name` — a name identifies exactly one - build, so a run cannot produce two jobs with one name and a dispatch cannot - be ambiguous. +- No two entries share a `name` — a name identifies exactly one build, so a run + cannot produce two jobs with one name and a dispatch cannot be ambiguous. - A flavour's entries agree on `srcpkg`, `binpkg` and `kernel_config`; those decide what the package *is*, and the entries differ only in where it goes. - A flavour's entries for one suite agree on `dkms`. The module set depends on which `-dkms` packages the target archive carries, so it varies between - suites — but a suite's Daily and Release must match, or the Daily is not - testing the module set the Release will ship. -- A flavour's entries of one `type` agree on `git_clone`, `branch_or_tag`, - `ref_strategy` and `tag_pattern`, so a forgotten suite cannot quietly ship a - different kernel from its siblings after a release ref bump. -- A flavour with a `Release` entry also has a `Daily` one, so nothing is - promoted that the daily build has not tested. The converse is allowed: a - flavour tracking a moving upstream is built daily and never released. + suites but not within one. +- A flavour's entries agree on `git_clone`, `branch_or_tag`, `ref_strategy` and + `tag_pattern`, so a forgotten suite cannot quietly ship a different kernel + from its siblings. - No `srcpkg` or `binpkg` is shared between flavours, and no two entries build the same `srcpkg` at the same `debian_revision`. -- Where a suite has both, its `Daily` revision is its `Release` revision plus a - trailing `~`. The build workflows pass their entry's own `debian_revision` through. A caller with no entry in hand can leave the `debian-revision` input empty, and the -`prepare-kernel-source` action looks up the `Daily` entry for the build name and -suite it was given (`resolve-matrix.py --field debian_revision`) and builds at -the revision the daily build would have used. +`prepare-kernel-source` action looks up the entry for the build name and suite +it was given (`resolve-matrix.py --field debian_revision`) and builds at the +revision the nightly build uses. Each entry has a distinct prepared-source artifact, Debusine child workspace, and S3 path keyed by `flavour + suite`. This prevents two flavours that both @@ -268,7 +257,7 @@ build, for example, `trixie` from consuming or publishing each other's inputs or outputs. They key on `flavour`, not on the build's `name`, so renaming a build never moves a published artifact. -Daily S3 outputs use these layouts, where `` is +S3 outputs use these layouts, where `` is `-`: ```text @@ -298,23 +287,25 @@ This document covers the CI generator. For the packaging internals: `debian/rule targets, the config fragment merge pipeline, DKMS module bundling and the produced package layout see [debian/README.md](debian/README.md). -Both branches below are taken in the caller, when the matrix is resolved: the -family from the entry's suite, the tail from which workflow is running. By the -time a build workflow starts, there is nothing left to decide. +The family branch below is taken in the caller, when the matrix is resolved, +from the entry's suite. The promotion branch is taken from the entry's own +`target_workspace`. By the time a build workflow starts, there is nothing left +to decide. ```mermaid flowchart LR IN["Matrix entries"] --> R{"resolve-matrix.py\n--family"} - R -->|"debian: trixie · forky"| DT{"Which caller"} + R -->|"debian: trixie · forky"| DEB["build-kernel-debian.yml\nGenerate source package\nSubmit with lib/build\nDebusine builds binaries"] R -->|"ubuntu: resolute"| UBU["build-kernel-ubuntu.yml\nbuild-kernel.sh in Docker\nBuild binary packages"] - DT -->|"daily.yml · pr-build.yml"| DEB["build-kernel-debian.yml\nGenerate source package\nSubmit with lib/build\nDebusine builds binaries"] - DT -->|"release.yml"| REL["release-kernel-debian.yml\nSame build, release tail"] - DEB --> S3["Download .deb files\nPublish to S3"] - REL --> QLI["Promote source and binaries\nto qli"] + DEB --> TW{"target_workspace\nset?"} + TW -->|yes| STG["Promote to qli-staging"] + TW -->|no| NONE["No archive"] UBU --> US3["Publish .deb files to S3"] + + STG -.->|"release.yml, later\nand builds nothing"| QLI["qli\nProduction APT repository"] ``` ## For CI Maintainers @@ -326,56 +317,56 @@ flowchart TD subgraph triggers[Triggers] A1["daily.yml\nScheduled full matrix"] A2["daily.yml\nManual: all or named builds"] - A5["pr-build.yml\nFull Daily matrix on every PR"] - A3["release.yml\nManual: all or named builds"] + A5["pr-build.yml\nFull matrix on every PR"] + A3["release.yml\nManual: version to release"] end subgraph matrix[Matrix entry points] - B1["configure-matrix\nDaily entries, split by family"] + B1["configure-matrix\nEntries, split by family"] B2["build-debian legs\nqcom-next · qcom-next-debug · qcom-arduino\nmainline · next / trixie · forky"] B5["build-ubuntu legs\nqcom-next / resolute"] - B3["configure-matrix\nRelease entries, Debian by construction"] - B4["build legs\nqcom-next · qcom-next-debug / trixie · forky"] + B3["configure-matrix\nEntries with a target_workspace"] end subgraph build[One build workflow per leg] - C2["prepare\nprepare-kernel-source action\nClone ref, run prepare-source.sh\nUpload kernel-srcpkg-flavour-suite"] + C2["prepare\nprepare-kernel-source action\nClone ref, run prepare-source.sh\nSkip the run if the version is published"] C3["build\ndebusine-build action"] C4["build\nbuild-kernel.sh in Docker"] C5["publish\nDownload .deb files, upload to S3"] - C6["release\nPromote to target workspace"] + C6["promote\nlib/release into target_workspace"] end subgraph outputs[Outputs] - D1["Daily S3 artifacts"] - D2["Release qli APT repository"] + D1["S3 artifacts"] + D3["qli-staging APT repository"] + D2["qli APT repository"] end A1 --> B1 A2 --> B1 A5 --> B1 - A3 --> B3 B1 --> B2 & B5 - B3 --> B4 B2 --> C2 B5 --> C2 - B4 --> C2 C2 --> C3 & C4 C3 --> C5 & C6 C4 --> D1 C5 --> D1 - C6 --> D2 + C6 --> D3 + A3 --> B3 + B3 -->|"promote only, no build"| D2 + D3 -.-> B3 ``` -Every leg runs every job drawn under it. `build-debian` and `build-ubuntu` legs -reach different build workflows, and `publish` and `release` belong to -different ones, so no leg starts a job it will skip. +A leg runs every job drawn under it except `promote`, which only entries naming +a `target_workspace` reach. `release.yml` touches none of the build column: it +reads what is already in `qli-staging`. ### Prepare stage ```mermaid flowchart LR - K["Matrix-selected kernel repository\nDaily: latest tag or branch tip\nRelease: pinned ref"] --> PS + K["Matrix-selected kernel repository\nlatest tag, branch tip, or pinned ref"] --> PS M["pkg-linux-qcom\ndebian/ and ci/ from this commit"] --> PS PS["prepare-source.sh\n\nInject debian/\nApply all config-available fragments plus any extras\nGenerate control, changelog, localversion, pkgversion"] --> TAR @@ -387,7 +378,7 @@ flowchart LR > Unix execute bits. Kernel build scripts require those permissions. The tar > archive preserves them between the prepare and build jobs. -### Debian Daily path +### Debian build path ```mermaid flowchart LR @@ -401,23 +392,28 @@ flowchart LR SUBMIT --> DEB["Debusine\nBuild binary packages"] DEB --> WS["Unique variant + suite workspace"] - subgraph publish[Daily publish job] + subgraph publish[publish job] WS --> APT["generate-apt-config\nchdist isolated APT environment\nDownload .deb files"] - APT --> S3["S3\nDaily package artifacts"] + APT --> S3["S3\npackage artifacts"] + end + + subgraph promote[promote job: only when target_workspace is set] + WS --> PROMOTE["lib/release\nStart package-publish"] + PROMOTE --> STG["qli-staging\nDebusine APT repository"] end ``` -### Debian Release path +### Release path + +Nothing is built. `release.yml` promotes a version that the nightly build +already put in `qli-staging`. ```mermaid flowchart LR - ART["kernel-srcpkg-variant-suite\nartifact"] --> GSP["generate-source-package\nProduce .dsc"] - GSP --> SUBMIT["lib/build\nSubmit source package to a unique\nDebusine CI child workspace"] - SUBMIT --> DEB["Debusine\nBuild binary packages"] - DEB --> WS["CI workspace\nsource and binary artifacts"] + STG["qli-staging\nsource and binary artifacts"] --> PROMOTE - subgraph release[Release job: Production GitHub environment] - WS --> PROMOTE["lib/release\nStart package-publish"] + subgraph release[promote job: Production GitHub environment] + PROMOTE["lib/release\nSRCPKG_VERSION = upstream-version + entry revision\nStart package-publish"] PROMOTE --> QLI["qli\nProduction Debusine APT repository"] end ``` @@ -433,7 +429,7 @@ flowchart LR BK["build-kernel.sh\n--skip-prepare\n--local-source\n--build-mode docker\ndpkg-buildpackage -b"] --> S3 end - S3["S3\nDaily package artifacts"] + S3["S3\npackage artifacts"] ``` `--skip-prepare` is safe because `prepare-source.sh` has already generated the @@ -490,21 +486,29 @@ out-of-tree module builds are required. ## Manual Builds Use **Actions** → **daily** → **Run workflow** for a one-off build, and -**Actions** → **release** to promote. Both are dispatched the same way: one -**Builds** field naming what to run. +**Actions** → **release** to promote a version that has already been built. +Both name what to act on in one **Builds** field. | Input | Default | Purpose | | --- | --- | --- | -| `builds` | `all` | `all` runs every entry of that workflow's delivery type. Otherwise a comma-separated list of build `name` values from `ci/build-matrix.yaml`, e.g. `qcom-next-trixie,qcom-next-debug-forky`. | +| `builds` | `all` | `all` selects every entry — for `release`, every entry that names a `target_workspace`. Otherwise a comma-separated list of build `name` values from `ci/build-matrix.yaml`, e.g. `qcom-next-trixie,qcom-next-debug-forky`. | Everything else about a build — its suite, flavour, kernel repository and ref, package names, config fragments, DKMS modules and Debian revision — comes from the entry, so there is nothing to retype and nothing to get wrong. Build names are free-text matrix values rather than a static dropdown, so adding a matrix entry never requires editing the workflow UI, and a name that matches no entry -of the delivery type fails the run with the list of names that do. `daily` -routes each selected entry to the workflow that builds its family, so one -dispatch can name Debian and Ubuntu builds together. +fails the run with the list of names that do. `daily` routes each selected +entry to the workflow that builds its family, so one dispatch can name Debian +and Ubuntu builds together. + +`release` carries two further inputs, because a promotion has to say what it is +promoting: + +| Input | Default | Purpose | +| --- | --- | --- | +| `upstream-version` | None, required | The version to release without its Debian revision, e.g. `7.2.0~rc7+20260821`. Each selected entry promotes this plus its own `debian_revision`. | +| `release-workspace` | `qli` | The Debusine workspace to promote into. | `daily` carries three further inputs, which the matrix deliberately says nothing about because they belong to a one-off validation run rather than to a @@ -517,12 +521,14 @@ them at their defaults: | `qcom-next-pr` | Empty | Advanced Qualcomm-only override: `qcom-next` PR numbers to merge before building. | | `kernel-topics-pr` | Empty | Advanced Qualcomm-only override: `kernel-topics` PR numbers to apply as patches. | -A `daily` dispatch is an artifact build and publishes to the daily S3 path; -Release promotion is performed only through `release.yml`. The build workflows -themselves (`build-kernel-debian.yml`, `build-kernel-ubuntu.yml`, -`release-kernel-debian.yml`) are `workflow_call` only and cannot be dispatched: -one run of each is one matrix entry, and a reusable workflow cannot fan itself -out over a list. +A `daily` dispatch publishes to S3 and, for an entry that names one, promotes +into that entry's `target_workspace` — unless it sets one of the advanced +overrides above, which suppress promotion because the kernel they build is not +the one the matrix describes. Promotion into the release workspace happens only +through `release.yml`. The build workflows themselves +(`build-kernel-debian.yml`, `build-kernel-ubuntu.yml`) are `workflow_call` only +and cannot be dispatched: one run of each is one matrix entry, and a reusable +workflow cannot fan itself out over a list. ## Configuration @@ -530,7 +536,7 @@ out over a list. | Variable | Purpose | | --- | --- | -| `ARTIFACT_S3_BUCKET` | S3 bucket for Daily Debian artifacts and Ubuntu build artifacts. | +| `ARTIFACT_S3_BUCKET` | S3 bucket for Debian and Ubuntu build artifacts. | | `DEBUSINE_HOST` | Production Debusine host. | | `DEBUSINE_SCOPE` | Debusine scope. | | `DEBUSINE_PARENT_WORKSPACE` | Parent workspace used to create per-run CI child workspaces. | @@ -540,54 +546,56 @@ out over a list. | Secret | Scope | Purpose | | --- | --- | --- | | `DEBUSINE_USER` | Repository | User for Debusine archive and signing-key access. | -| `DEBUSINE_TOKEN` | Repository | Token for Debusine build and artifact operations. | -| `DEBUSINE_RELEASE_TOKEN` | Production environment | Token used only to promote Release artifacts to `qli`. | +| `DEBUSINE_TOKEN` | Repository | Token for Debusine build and artifact operations, including the nightly promotion into `qli-staging`. | +| `DEBUSINE_RELEASE_TOKEN` | Production environment | Token used only to promote into the release workspace. | -The Debian build and Release jobs select the **Production** GitHub environment. -This makes environment-scoped release credentials available to the promotion job -and keeps production approval controls in the workflow path. +The `build` and `promote` jobs of `build-kernel-debian.yml` select the +**Staging** GitHub environment; the promotion job of `release.yml` selects +**Production**. That split is what lets the nightly build promote into +`qli-staging` unattended while a release into `qli` still passes through the +production approval gate. `DEBUSINE_TOKEN` therefore needs write access to +`qli-staging`. ## Maintaining the Matrix To add a kernel variant: 1. Add one entry to `builds` per package the variant should produce: one per - Daily suite and one per Release suite, each spelling out all of its own - fields, and each with a `name` unique within its delivery type. Do not rely - on another variant's values. + suite, each spelling out all of its own fields, and each with a `name` + unique across the matrix. Do not rely on another variant's values. 2. Give them all the same `flavour`, distinct from every other flavour's — it becomes the LOCALVERSION suffix, so this is what lets the new kernel install - alongside the existing ones. Keep `srcpkg`, `binpkg` and `kernel_config` - identical across every entry for the flavour, and the ref fields identical - across its entries of one `type`. `resolve-matrix.py` rejects the matrix if - they drift apart. -3. Use `latest_tag` with a dated tag glob or `branch_tip` for Daily. Use - `pinned_ref` for Release, and update that ref through a reviewed PR. + alongside the existing ones. Keep `srcpkg`, `binpkg`, `kernel_config` and + the ref fields identical across every entry for the flavour. + `resolve-matrix.py` rejects the matrix if they drift apart. +3. Use `latest_tag` with a dated tag glob, or `branch_tip`, to track a moving + upstream; `pinned_ref` freezes the variant on one ref. 4. Give the variant distinct `srcpkg` and `binpkg` values. Set - `target_workspace` on each Release entry. -5. Give each entry a `debian_revision`: the Daily one is the Release one for - the same suite plus a trailing `~`. + `target_workspace` on the Debian entries that should be installable from an + archive, and leave it off the ones built only for early warning. +5. Give each entry a `debian_revision` that sorts where its suite belongs + relative to the others and collides with no other entry building the same + `srcpkg`. 6. Confirm suite-family routing: Debian suites use Debusine; Ubuntu suites use the Docker path. -7. Run a filtered Daily validation for the new variant, then its full Daily and - Release flows. +7. Run a filtered daily validation for the new variant, then a full daily run. + Nothing further is needed to release it: once its packages are in the + staging workspace, `release.yml` can promote them. To add a new suite (for an existing or new variant): -1. Add one entry per delivery type the suite should get, copying the variant's - existing entry for that type and changing `suite` and `debian_revision`. +1. Add one entry for the suite, copying one of the variant's existing entries + and changing `suite` and `debian_revision`. 2. Choose the revision so the suite sorts where it belongs relative to the - others for the same delivery type (see the ordering discussion in - [Overview](#overview)), and so it does not collide with another entry - building the same `srcpkg`. + others (see the ordering discussion in [Overview](#overview)), and so it + does not collide with another entry building the same `srcpkg`. -No workflow dispatch choices need to be updated: the Daily and Release +No workflow dispatch choices need to be updated: the daily and release dispatches take build names as free text, so a new entry is dispatchable by name, and is picked up by `all`, as soon as it is merged. -Run `ci/scripts/resolve-matrix.py --type Daily` and `--type Release` locally to -validate a matrix change before pushing it; both validate the whole document, -so either one catches a mistake in the other's entries. +Run `ci/scripts/resolve-matrix.py` locally to validate a matrix change before +pushing it; it validates the whole document regardless of what it selects. ## Contributing diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index 784809ea..dcdaf503 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -3,19 +3,18 @@ # # Kernel delivery matrix. # -# One entry in "builds" is one generated package: a single name, a single -# delivery type, and a single suite. The only list-valued fields are -# kernel_config and dkms, which name the fragments and the out-of-tree modules -# that go into that one package, so an entry needs no expansion — ci/scripts/ -# resolve-matrix.py validates the document, filters it, and hands the matching -# entries straight to the workflow matrix. +# One entry in "builds" is one generated package: a single name for a single +# suite. The only list-valued fields are kernel_config and dkms, which name the +# fragments and the out-of-tree modules that go into that one package, so an +# entry needs no expansion — ci/scripts/resolve-matrix.py validates the +# document, filters it, and hands the matching entries straight to the workflow +# matrix. # # Entries are written out in full rather than sharing YAML anchors, so each one # can be read, grepped, and changed on its own. resolve-matrix.py enforces the # consistency that duplication would otherwise put at risk: a flavour's entries -# must agree on srcpkg, binpkg and kernel_config, its entries of one delivery -# type must agree on the kernel ref they build, and its entries for one suite -# must agree on dkms. +# must agree on srcpkg, binpkg, kernel_config and the kernel ref they build, +# and its entries for one suite must agree on dkms. # # The two identifiers are not the same thing. flavour is the kernel's own # identity: it becomes the LOCALVERSION suffix, so qcom-next and @@ -23,31 +22,36 @@ # other, and every suite building one flavour produces the same kernel release. # A build's name is what CI calls it: the Actions job name, and what a manual # dispatch asks for by name. It never reaches a package name or version, and -# never a published path. It must be unique within a delivery type, so a Daily -# and the Release that supersedes it share one. It may not be "all", which the -# daily and release dispatch forms read as every entry of their type. +# never a published path. It must be unique, and may not be "all", which the +# daily and release dispatch forms read as every entry. # # dkms varies by suite because it depends on which -dkms packages the # target archive carries: the Debian suites bundle camx and iris-vpu, which # Ubuntu does not package. # -# A flavour tracking a moving upstream is Daily only. A Release entry must name -# an immutable ref, and a branch tip or a tag cut every night is not one, so -# mainline, next and the topic branches are built and published daily and never -# promoted. A Release entry still requires the Daily that tests it. +# Every entry is built nightly. An entry that names a target_workspace is also +# promoted into it once the build succeeds, so the archive carries the artifact +# the nightly build tested rather than a rebuild of the same ref. Promoting +# onward from there into a release workspace builds nothing and is driven by +# release.yml, which names the version to promote; nothing here pins a release +# ref. mainline, next and the topic branches name no target_workspace: they are +# early-warning builds, published to S3 and nowhere else. # -# debian_revision is stated outright, not derived. Daily revisions end in "~" -# so they sort below the Release revision for the same suite; the rest of the -# revision places the suite where it belongs relative to the other suites (see -# the ordering discussion in README.md). No two entries may produce the same -# srcpkg at the same debian_revision. +# Only a Debian suite may name a target_workspace. Promotion runs through +# Debusine, which does not build the Ubuntu family. +# +# debian_revision is stated outright, not derived. The trailing "~" marks a +# version as unreleased, and it is carried into the archive as built: what +# release.yml promotes is the tested artifact, not a rebuild at a tidier +# version. The rest of the revision places the suite where it belongs relative +# to the other suites (see the ordering discussion in README.md). No two +# entries may produce the same srcpkg at the same debian_revision. builds: # --------------------------------------------------------------------------- # qcom-next — the standard kernel. # --------------------------------------------------------------------------- - name: qcom-next-trixie - type: Daily suite: trixie flavour: qcom-next git_clone: https://github.com/qualcomm-linux/kernel @@ -63,9 +67,9 @@ builds: - iris-vpu - audioreach debian_revision: '0qli1~bpo13+1~' + target_workspace: qli-staging - name: qcom-next-forky - type: Daily suite: forky flavour: qcom-next git_clone: https://github.com/qualcomm-linux/kernel @@ -81,9 +85,9 @@ builds: - iris-vpu - audioreach debian_revision: '0qli1~' + target_workspace: qli-staging - name: qcom-next-resolute - type: Daily suite: resolute flavour: qcom-next git_clone: https://github.com/qualcomm-linux/kernel @@ -98,48 +102,11 @@ builds: - kgsl debian_revision: '0qli1~26.04.1~' - - name: qcom-next-trixie - type: Release - suite: trixie - flavour: qcom-next - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next-7.2-rc7-20260821 - ref_strategy: pinned_ref - srcpkg: linux-qcom-next - binpkg: linux-image-qcom-next - kernel_config: [] - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1~bpo13+1' - target_workspace: qli - - - name: qcom-next-forky - type: Release - suite: forky - flavour: qcom-next - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next-7.2-rc7-20260821 - ref_strategy: pinned_ref - srcpkg: linux-qcom-next - binpkg: linux-image-qcom-next - kernel_config: [] - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1' - target_workspace: qli - # --------------------------------------------------------------------------- # qcom-next-debug — the same kernel ref built with the in-tree debug # fragments, packaged separately so it installs alongside qcom-next. # --------------------------------------------------------------------------- - name: qcom-next-debug-trixie - type: Daily suite: trixie flavour: qcom-next-debug git_clone: https://github.com/qualcomm-linux/kernel @@ -157,9 +124,9 @@ builds: - iris-vpu - audioreach debian_revision: '0qli1~bpo13+1~' + target_workspace: qli-staging - name: qcom-next-debug-forky - type: Daily suite: forky flavour: qcom-next-debug git_clone: https://github.com/qualcomm-linux/kernel @@ -177,54 +144,15 @@ builds: - iris-vpu - audioreach debian_revision: '0qli1~' - - - name: qcom-next-debug-trixie - type: Release - suite: trixie - flavour: qcom-next-debug - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next-7.2-rc7-20260821 - ref_strategy: pinned_ref - srcpkg: linux-qcom-next-debug - binpkg: linux-image-qcom-next-debug - kernel_config: - - intree:arch/arm64/configs/qcom_debug.config - - intree:kernel/configs/debug.config - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1~bpo13+1' - target_workspace: qli - - - name: qcom-next-debug-forky - type: Release - suite: forky - flavour: qcom-next-debug - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next-7.2-rc7-20260821 - ref_strategy: pinned_ref - srcpkg: linux-qcom-next-debug - binpkg: linux-image-qcom-next-debug - kernel_config: - - intree:arch/arm64/configs/qcom_debug.config - - intree:kernel/configs/debug.config - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1' - target_workspace: qli + target_workspace: qli-staging # --------------------------------------------------------------------------- # qcom-arduino — the Arduino hardware-enablement topic branch. # - # Daily only: a topic branch has no immutable ref to promote from. + # No target_workspace: a topic branch is built for early warning, not for + # anyone to install from an archive. # --------------------------------------------------------------------------- - name: qcom-arduino-trixie - type: Daily suite: trixie flavour: qcom-arduino git_clone: https://github.com/qualcomm-linux/kernel-topics @@ -240,7 +168,6 @@ builds: debian_revision: '0qli~bpo13+1~' - name: qcom-arduino-forky - type: Daily suite: forky flavour: qcom-arduino git_clone: https://github.com/qualcomm-linux/kernel-topics @@ -258,11 +185,11 @@ builds: # --------------------------------------------------------------------------- # mainline — Linus's tree, tracked for early warning of upstream breakage. # - # Daily only, and no DKMS: the out-of-tree modules target the Qualcomm tree, - # and a listed module is a presence contract that would fail the build here. + # No target_workspace, and no DKMS: the out-of-tree modules target the + # Qualcomm tree, and a listed module is a presence contract that would fail + # the build here. # --------------------------------------------------------------------------- - name: mainline-trixie - type: Daily suite: trixie flavour: mainline git_clone: https://github.com/torvalds/linux @@ -275,7 +202,6 @@ builds: debian_revision: '0qli~bpo13+1~' - name: mainline-forky - type: Daily suite: forky flavour: mainline git_clone: https://github.com/torvalds/linux @@ -291,10 +217,10 @@ builds: # next — linux-next, the integration tree, one step ahead of mainline. # # next-YYYYMMDD tags are cut daily, so latest_tag resolves the newest of - # them the same way qcom-next-* does. Daily only, and no DKMS, as above. + # them the same way qcom-next-* does. No target_workspace and no DKMS, as + # above. # --------------------------------------------------------------------------- - name: next-trixie - type: Daily suite: trixie flavour: next git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git @@ -308,7 +234,6 @@ builds: debian_revision: '0qli~bpo13+1~' - name: next-forky - type: Daily suite: forky flavour: next git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index 7afa7d3f..b9a93d47 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -4,27 +4,30 @@ """Validate and select entries from the kernel delivery matrix. ci/build-matrix.yaml holds one entry per generated package: a single named -build, one delivery type, and one suite. Nothing here -expands or derives anything -- the matrix is already flat, and each entry -states its own debian_revision. This script validates the whole document, -selects the entries a caller asked for, and prints them. +build for a single suite. Nothing here expands or derives anything -- the +matrix is already flat, and each entry states its own debian_revision. This +script validates the whole document, selects the entries a caller asked for, +and prints them. + +There is one kind of entry. Every entry is built nightly, and an entry that +names a target_workspace is also promoted into it, so the archive carries the +artifact the nightly build tested rather than a second build of the same ref. +Promoting that artifact onward to a release workspace is a separate step that +builds nothing; see release.yml. The document is validated in full on every invocation, not just the selected -entries, so a typo in a Release entry fails the daily build too rather than -lying in wait until someone runs a release. +entries, so a typo in an entry nobody selected fails the run that would have +built its siblings rather than lying in wait. Usage: - ci/scripts/resolve-matrix.py --type Daily - ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie - ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie,qcom-next-forky - ci/scripts/resolve-matrix.py --type Release --flavour qcom-next - ci/scripts/resolve-matrix.py --type Daily --family ubuntu --allow-empty - ci/scripts/resolve-matrix.py --type Daily --build qcom-next-trixie \\ - --field debian_revision + ci/scripts/resolve-matrix.py + ci/scripts/resolve-matrix.py --build qcom-next-trixie + ci/scripts/resolve-matrix.py --build qcom-next-trixie,qcom-next-forky + ci/scripts/resolve-matrix.py --flavour qcom-next + ci/scripts/resolve-matrix.py --family ubuntu --allow-empty + ci/scripts/resolve-matrix.py --build qcom-next-trixie --field debian_revision Options: - --type TYPE Delivery type to select (Daily or Release). - Required. --build NAMES Select only these builds by name, comma-separated. A name identifies one build, so this is the way to ask for a specific set of them. @@ -47,9 +50,9 @@ relative to CWD). Filters combine: an entry must match every filter given. Every name in a - filter must match at least one entry of the selected type, so a typo or a - stale name fails instead of quietly narrowing the build set. That check is - per filter, so --allow-empty still rejects a name that matches nothing. + filter must match at least one entry, so a typo or a stale name fails + instead of quietly narrowing the build set. That check is per filter, so + --allow-empty still rejects a name that matches nothing. Output: Without --field, a compact JSON array of the selected entries, ready for a @@ -84,7 +87,6 @@ DEFAULT_MATRIX_FILE = "ci/build-matrix.yaml" -DELIVERY_TYPES = ("Daily", "Release") REF_STRATEGIES = ("latest_tag", "branch_tip", "pinned_ref") # Which build path a suite takes. Debian-family suites are built by Debusine; @@ -101,16 +103,9 @@ def family_for(suite): """Return the build family a suite belongs to.""" return "debian" if suite in DEBIAN_SUITES else "ubuntu" -# A delivery type constrains how its kernel ref is chosen: a Daily build tracks -# something moving, a Release build is pinned to an immutable ref. -REF_STRATEGIES_FOR_TYPE = { - "Daily": ("latest_tag", "branch_tip"), - "Release": ("pinned_ref",), -} REQUIRED_STRING_FIELDS = ( "name", - "type", "suite", "flavour", "git_clone", @@ -147,22 +142,20 @@ def family_for(suite): # Fields a flavour may vary between suites but not within one. The out-of-tree # module set depends on which -dkms packages the target archive has, so -# it is a property of the flavour in a suite rather than of the flavour. A -# suite's Daily and Release must still agree: a Daily that bundles a different -# module set from the Release it precedes is not testing what will ship. +# it is a property of the flavour in a suite rather than of the flavour. SUITE_IDENTITY_FIELDS = ("dkms",) -# Fields deciding which kernel tree is built. All entries for one flavour and -# delivery type build the same source, so a stale suite cannot quietly ship a -# different kernel from its siblings. +# Fields deciding which kernel tree is built. All entries for one flavour build +# the same source, so a stale suite cannot quietly ship a different kernel from +# its siblings. REF_FIELDS = ("git_clone", "branch_or_tag", "ref_strategy", "tag_pattern") NAME_RE = re.compile(r"^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$") # The dispatch forms of daily.yml and release.yml take one builds field, where -# "all" means every entry of the delivery type and anything else is a list of -# names. A build actually called all would be unreachable through them, so the -# matrix may not define one. +# "all" means every entry and anything else is a list of names. A build +# actually called all would be unreachable through them, so the matrix may not +# define one. RESERVED_NAMES = ("all",) # A Debian revision: no hyphen (that would start a new revision component) and @@ -285,20 +278,9 @@ def check_entry(entry, report): check_kernel_config(entry, report) check_dkms(entry, report) - delivery_type = entry.get("type") - if delivery_type not in DELIVERY_TYPES: - report("type must be Daily or Release") - ref_strategy = entry.get("ref_strategy") if ref_strategy not in REF_STRATEGIES: report("ref_strategy must be " + ", ".join(REF_STRATEGIES)) - elif delivery_type in REF_STRATEGIES_FOR_TYPE: - allowed = REF_STRATEGIES_FOR_TYPE[delivery_type] - if ref_strategy not in allowed: - report( - f"{delivery_type} entries must use " - + " or ".join(f"ref_strategy={s}" for s in allowed) - ) if ref_strategy == "latest_tag": if not entry.get("tag_pattern"): @@ -306,36 +288,24 @@ def check_entry(entry, report): elif "tag_pattern" in entry: report("tag_pattern is only valid with ref_strategy=latest_tag") - if delivery_type == "Release": - if not entry.get("target_workspace"): - report("missing or invalid target_workspace") - # Promotion runs through Debusine, and only the Debian family is built - # there. An Ubuntu Release entry would build the package and then have - # no workspace to promote it from, publishing to the daily S3 path and - # reporting success without ever releasing anything. + # Promotion runs through Debusine, and only the Debian family is built + # there. An Ubuntu entry naming a target_workspace would build the package + # and then have no workspace to promote it from, publishing to the S3 path + # and reporting success without the archive ever gaining anything. + if entry.get("target_workspace"): suite = entry.get("suite") if isinstance(suite, str) and family_for(suite) != "debian": report( - f"Release entries must target a Debian suite, not {suite}; " - "promotion runs through Debusine, which builds " - + ", ".join(DEBIAN_SUITES) + f"target_workspace needs a Debian suite, not {suite}; promotion " + "runs through Debusine, which builds " + ", ".join(DEBIAN_SUITES) ) - elif "target_workspace" in entry: - report("target_workspace is only valid for Release") revision = entry.get("debian_revision") - if isinstance(revision, str) and revision: - if not REVISION_RE.match(revision): - report( - f'debian_revision "{revision}" is not a valid Debian revision ' - "(letters, digits, and . + ~ only, starting with a letter or digit)" - ) - # A trailing ~ sorts a version below the same version without it, so a - # Daily always sorts below the Release it will be superseded by. - elif delivery_type == "Daily" and not revision.endswith("~"): - report(f'debian_revision "{revision}" must end in ~ for a Daily entry') - elif delivery_type == "Release" and revision.endswith("~"): - report(f'debian_revision "{revision}" must not end in ~ for a Release entry') + if isinstance(revision, str) and revision and not REVISION_RE.match(revision): + report( + f'debian_revision "{revision}" is not a valid Debian revision ' + "(letters, digits, and . + ~ only, starting with a letter or digit)" + ) def describe(entry, index): @@ -344,7 +314,7 @@ def describe(entry, index): return f"entry {index}" parts = [ str(entry[field]) - for field in ("name", "type", "suite") + for field in ("name", "suite") if isinstance(entry.get(field), str) ] return f"entry {index} ({'/'.join(parts)})" if parts else f"entry {index}" @@ -361,9 +331,8 @@ def check_consistency(builds, errors): only checked for the one thing it is used for: identifying that build uniquely. """ - by_leg = defaultdict(list) + by_name = defaultdict(list) by_flavour = defaultdict(list) - by_flavour_type = defaultdict(list) by_flavour_suite = defaultdict(list) by_package_version = defaultdict(list) flavours_by_package = defaultdict(set) @@ -372,15 +341,13 @@ def check_consistency(builds, errors): if not isinstance(entry, dict): continue flavour = entry.get("flavour") - delivery_type = entry.get("type") suite = entry.get("suite") if not isinstance(flavour, str): continue by_flavour[flavour].append(entry) - by_flavour_type[(flavour, delivery_type)].append(entry) by_flavour_suite[(flavour, suite)].append(entry) - by_leg[(delivery_type, entry.get("name"))].append(entry) + by_name[entry.get("name")].append(entry) for field in ("srcpkg", "binpkg"): if isinstance(entry.get(field), str) and entry[field]: @@ -392,17 +359,13 @@ def check_consistency(builds, errors): by_package_version[(entry["srcpkg"], entry["debian_revision"])].append(entry) # A name identifies one build: it is the Actions job name and what a - # workflow dispatch asks for. It only has to be unique within a delivery - # type, because a run only ever resolves one type, and that lets a Daily and - # its Release share a name. Two entries sharing both would give a run two + # workflow dispatch asks for. Two entries sharing one would give a run two # identically named jobs and make the dispatch filter ambiguous. - for (delivery_type, name), entries in sorted( - by_leg.items(), key=lambda item: str(item[0]) - ): + for name, entries in sorted(by_name.items(), key=lambda item: str(item[0])): if len(entries) > 1: suites = ", ".join(sorted(str(e.get("suite")) for e in entries)) errors.append( - f"name {name} is used by {len(entries)} {delivery_type} entries " + f"name {name} is used by {len(entries)} entries " f"(suites {suites}); a name identifies exactly one build" ) @@ -414,15 +377,6 @@ def check_consistency(builds, errors): f"flavour {flavour} must use one {field} across all its " "entries (got " + ", ".join(sorted(values)) + ")" ) - # A Release must be preceded by the Daily that tests it, but the - # converse does not hold: a flavour tracking a moving upstream has - # nothing immutable to pin, so it is built daily and never promoted. - types = {entry.get("type") for entry in entries} - if "Release" in types and "Daily" not in types: - errors.append( - f"flavour {flavour} has a Release entry but no Daily entry; " - "a release must be preceded by the daily build that tests it" - ) for (flavour, suite), entries in sorted( by_flavour_suite.items(), key=lambda item: str(item[0]) @@ -435,16 +389,14 @@ def check_consistency(builds, errors): f"{suite} entries (got " + ", ".join(sorted(values)) + ")" ) - for (flavour, delivery_type), entries in sorted( - by_flavour_type.items(), key=lambda item: str(item[0]) - ): + for flavour, entries in sorted(by_flavour.items()): for field in REF_FIELDS: values = {entry.get(field) for entry in entries} if len(values) > 1: rendered = ", ".join(sorted(str(v) for v in values)) errors.append( f"flavour {flavour} must build one {field} across its " - f"{delivery_type} entries (got {rendered})" + f"entries (got {rendered})" ) # Two flavours sharing a package name would overwrite each other in the @@ -465,26 +417,6 @@ def check_consistency(builds, errors): f"suites {suites}; each entry needs a revision of its own" ) - # A suite's Daily and Release differ only by the Daily's trailing ~, so the - # Daily reliably sorts below the Release that supersedes it. - for flavour, entries in sorted(by_flavour.items()): - revisions = { - (entry.get("type"), entry.get("suite")): entry.get("debian_revision") - for entry in entries - } - for (delivery_type, suite), revision in sorted( - revisions.items(), key=lambda item: str(item[0]) - ): - if delivery_type != "Daily" or not isinstance(revision, str): - continue - release = revisions.get(("Release", suite)) - if isinstance(release, str) and revision != release + "~": - errors.append( - f"flavour {flavour} suite {suite}: Daily " - f'debian_revision "{revision}" must be the Release revision ' - f'"{release}" with a trailing ~' - ) - def validate(builds): """Return every problem found in the matrix, as a list of messages.""" @@ -553,7 +485,7 @@ def parse_filter(value): return [name.strip() for name in value.split(",") if name.strip()] -def select(builds, delivery_type, filters): +def select(builds, filters): """Return the entries matching the requested filters, in matrix order. filters maps a field name to the list of values allowed for it, or to @@ -562,51 +494,47 @@ def select(builds, delivery_type, filters): return [ entry for entry in builds - if entry["type"] == delivery_type - and all( + if all( wanted is None or entry[field] in wanted for field, wanted in filters.items() ) ] -def unmatched_filters(builds, delivery_type, filters): - """Names asked for that no entry of this delivery type offers. +def unmatched_filters(builds, filters): + """Names asked for that no entry offers. family is exempt: it routes a selection to a build path rather than naming something in the matrix, argparse already restricts it to a real family, - and a type having no builds on one path is an ordinary state of the matrix - rather than a mistake in the request. + and a matrix having no builds on one path is an ordinary state of it rather + than a mistake in the request. """ missing = [] for field, wanted in filters.items(): if wanted is None or field == "family": continue - available = { - entry[field] for entry in builds if entry["type"] == delivery_type - } + available = {entry[field] for entry in builds} for name in wanted: if name not in available: missing.append( - f"no {delivery_type} entry has {field} {name} " + f"no entry has {field} {name} " f"(available: {', '.join(sorted(available))})" ) return missing -def describe_selection(delivery_type, filters): +def describe_selection(filters): """Render the active filters for an error message.""" - parts = [f"type={delivery_type}"] - parts += [ + parts = [ f"{field}={','.join(wanted)}" for field, wanted in filters.items() if wanted is not None ] - return " ".join(parts) + return " ".join(parts) if parts else "the whole matrix" def for_workflow(entry): - """Shape one entry the way build-kernel-deb.yml's inputs expect it.""" + """Shape one entry the way the build workflows' inputs expect it.""" return { **entry, "kernel_config": ",".join(entry["kernel_config"]), @@ -619,9 +547,6 @@ def main(): description="Validate and select entries from the kernel delivery matrix.", epilog="See the module docstring in this file for full documentation.", ) - parser.add_argument( - "--type", required=True, choices=DELIVERY_TYPES, help="delivery type to select" - ) parser.add_argument( "--build", default="", @@ -661,19 +586,19 @@ def main(): "suite": parse_filter(args.suite), "family": parse_filter(args.family), } - what = describe_selection(args.type, filters) + what = describe_selection(filters) # Report a name that matches nothing before reporting an empty selection: - # "no Daily entry has name qcom-next" says what to fix, where - # "no entries found" leaves the reader to work out which filter was wrong. - unmatched = unmatched_filters(builds, args.type, filters) + # "no entry has name qcom-nxt" says what to fix, where "no entries found" + # leaves the reader to work out which filter was wrong. + unmatched = unmatched_filters(builds, filters) if unmatched: sys.exit( f"ERROR: Nothing to build for {what}:\n" + "\n".join(f" - {problem}" for problem in unmatched) ) - selected = select(builds, args.type, filters) + selected = select(builds, filters) if not selected: # A caller splitting one dispatch across both build paths asks each # family for the same selection, and one of them legitimately has From f6d82ab0ffa4ca52ae4332156964a0c2d511fc00 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 36/71] ci: drop the debug-build input Setting it broke the build. The input threaded from daily.yml through both family workflows into prepare-kernel-source, which turned it into --debug on the prepare-source.sh command line -- an option prepare-source.sh does not have, and whose argument parser rejects an unknown one outright. Anyone ticking the box on a dispatch got "Unknown option: --debug" and a failed prepare job, so the toggle has only ever been a way to break a run. Nothing is lost by removing it. A debug kernel is a flavour, not a run-time switch: qcom-next-debug builds the same ref with the in-tree debug fragments and packages it separately, so it installs alongside qcom-next and is built and published every night like anything else in the matrix. That is what the input's own help text already pointed people at. The promote job's guard loses a clause with it and keeps its meaning: a dispatch that overrides what gets built still promotes nowhere, and there are now two ways to do that rather than three. Signed-off-by: Christopher Obbard --- .../actions/prepare-kernel-source/action.yml | 7 +------ .github/workflows/build-kernel-debian.yml | 6 ------ .github/workflows/build-kernel-ubuntu.yml | 5 ----- .github/workflows/daily.yml | 7 ------- README.md | 17 ++++++++--------- 5 files changed, 9 insertions(+), 33 deletions(-) diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index 3811def2..5b2f45df 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -77,7 +77,7 @@ inputs: description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' default: '' debian-revision: - description: 'Debian revision override (empty takes the Daily revision the matrix gives this build)' + description: 'Debian revision override (empty takes the revision the matrix gives this build)' default: '' localversion: description: 'LOCALVERSION override (auto-derived from the resolved ref if empty)' @@ -85,9 +85,6 @@ inputs: kver-extra: description: 'Extra package-version suffix (e.g. -ci42)' default: '' - debug-build: - description: 'Enable the debug build configuration' - default: 'false' qcom-next-pr: description: 'qcom-next PR numbers to merge, space-separated' default: '' @@ -387,7 +384,6 @@ runs: KVER_EXTRA_INPUT: ${{ inputs.kver-extra }} KERNEL_CONFIG_INPUT: ${{ inputs.kernel-config }} DKMS_INPUT: ${{ inputs.dkms }} - DEBUG_BUILD_INPUT: ${{ inputs.debug-build }} KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} DISTRO: ${{ steps.env.outputs.distro }} DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} @@ -426,7 +422,6 @@ runs: # Always passed, unlike the optional inputs above: an empty list is a # genuine "bundle nothing" rather than a request for some default. ARGS+=(--dkms "$DKMS_INPUT") - [[ "$DEBUG_BUILD_INPUT" == "true" ]] && ARGS+=(--debug) docker run -i --rm \ -v "$PWD:$PWD" --workdir="$PWD" \ diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 0eac719a..b78adbf9 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -87,10 +87,6 @@ on: description: 'Extra suffix appended to the package version' type: string default: '' - debug-build: - description: 'Enable the debug build configuration' - type: boolean - default: false debusine-parent-workspace: description: 'Parent Debusine workspace for CI child workspace creation' type: string @@ -175,7 +171,6 @@ jobs: debian-revision: ${{ inputs.debian-revision }} localversion: ${{ inputs.localversion }} kver-extra: ${{ inputs.kver-extra }} - debug-build: ${{ inputs.debug-build }} qcom-next-pr: ${{ inputs.qcom-next-pr }} kernel-topics-pr: ${{ inputs.kernel-topics-pr }} @@ -391,7 +386,6 @@ jobs: needs: build if: >- ${{ inputs.target-workspace != '' - && !inputs.debug-build && inputs.qcom-next-pr == '' && inputs.kernel-topics-pr == '' }} runs-on: ubuntu-latest diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 33a4b49f..eb972300 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -81,10 +81,6 @@ on: description: 'Extra suffix appended to the package version' type: string default: '' - debug-build: - description: 'Enable the debug build configuration' - type: boolean - default: false qcom-next-pr: description: 'qcom-next PR numbers to merge (e.g. "42 43")' type: string @@ -132,7 +128,6 @@ jobs: debian-revision: ${{ inputs.debian-revision }} localversion: ${{ inputs.localversion }} kver-extra: ${{ inputs.kver-extra }} - debug-build: ${{ inputs.debug-build }} qcom-next-pr: ${{ inputs.qcom-next-pr }} kernel-topics-pr: ${{ inputs.kernel-topics-pr }} diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 1272abb0..58215300 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -40,11 +40,6 @@ on: # overrides, so they are inputs rather than entry fields. They apply to # every selected build, and a scheduled run leaves them at their # defaults. - debug-build: - description: 'Advanced: enable the debug build configuration' - required: false - default: false - type: boolean qcom-next-pr: description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' required: false @@ -167,7 +162,6 @@ jobs: target-workspace: ${{ matrix.target_workspace || '' }} # Dispatch-only validation overrides. A scheduled run carries no inputs, # so each falls back to the off value and builds the matrix as written. - debug-build: ${{ inputs.debug-build || false }} qcom-next-pr: ${{ inputs.qcom-next-pr || '' }} kernel-topics-pr: ${{ inputs.kernel-topics-pr || '' }} secrets: @@ -201,6 +195,5 @@ jobs: debian-revision: ${{ matrix.debian_revision }} localversion: ${{ matrix.localversion || '' }} kver-extra: ${{ matrix.kver_extra || '' }} - debug-build: ${{ inputs.debug-build || false }} qcom-next-pr: ${{ inputs.qcom-next-pr || '' }} kernel-topics-pr: ${{ inputs.kernel-topics-pr || '' }} diff --git a/README.md b/README.md index ce384608..88a87605 100644 --- a/README.md +++ b/README.md @@ -161,10 +161,10 @@ promotion when it does. A run that skips this way is green: nothing was wrong, there was simply nothing new upstream. Three kinds of run deliberately promote nowhere: a PR build, which never -forwards a `target_workspace`; a dispatch setting any of the advanced -validation overrides (`debug-build`, `qcom-next-pr`, `kernel-topics-pr`), -whose kernel is by definition not the one the matrix describes; and any entry -that names no workspace. +forwards a `target_workspace`; a dispatch setting either of the advanced +validation overrides (`qcom-next-pr`, `kernel-topics-pr`), whose kernel is by +definition not the one the matrix describes; and any entry that names no +workspace. ### Release @@ -510,14 +510,13 @@ promoting: | `upstream-version` | None, required | The version to release without its Debian revision, e.g. `7.2.0~rc7+20260821`. Each selected entry promotes this plus its own `debian_revision`. | | `release-workspace` | `qli` | The Debusine workspace to promote into. | -`daily` carries three further inputs, which the matrix deliberately says -nothing about because they belong to a one-off validation run rather than to a -delivery target. They apply to every selected build, and a scheduled run leaves -them at their defaults: +`daily` carries two further inputs, which the matrix deliberately says nothing +about because they belong to a one-off validation run rather than to a delivery +target. They apply to every selected build, and a scheduled run leaves them at +their defaults: | Input | Default | Purpose | | --- | --- | --- | -| `debug-build` | `false` | Advanced debug configuration toggle. For a lasting debug kernel, use the `qcom-next-debug` flavour instead. | | `qcom-next-pr` | Empty | Advanced Qualcomm-only override: `qcom-next` PR numbers to merge before building. | | `kernel-topics-pr` | Empty | Advanced Qualcomm-only override: `kernel-topics` PR numbers to apply as patches. | From 34d6c9ec2625b48f33f62c70c30cbd5dc5e52b95 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 37/71] ci: drop the PR-override inputs from the workflows qcom-next-pr and kernel-topics-pr were dispatch fields on daily.yml, threaded through both family workflows to reach prepare-kernel-source. They described a kernel the matrix does not: a tree with pull requests merged or patches applied on top of the resolved ref, built at the same version as the nightly that has neither. Now that a build promotes into an archive, a one-off validation run reaching a dispatch field is a way to put an unreviewed tree in front of everyone who installs from it, so the fields go and the promote guard loses the clauses that existed to catch them. The action keeps both inputs and the steps that act on them. Merging a PR before building is still a real thing to want; it is simply no longer something a dispatch form offers, and a caller that wants it has to say so deliberately. daily.yml is left with one input, builds, which is the whole of what a manual run has to decide. The promote guard is now just target-workspace: pr-build.yml is the caller that names none, so a pull request's kernel still reaches no archive. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 25 ++++------------------ .github/workflows/build-kernel-ubuntu.yml | 10 --------- .github/workflows/daily.yml | 25 +--------------------- README.md | 26 ++++++----------------- 4 files changed, 12 insertions(+), 74 deletions(-) diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index b78adbf9..71f5774c 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -99,14 +99,6 @@ on: reach an archive. type: string default: '' - qcom-next-pr: - description: 'qcom-next PR numbers to merge (e.g. "42 43")' - type: string - default: '' - kernel-topics-pr: - description: 'kernel-topics PR numbers to apply as patches' - type: string - default: '' secrets: DEBUSINE_USER: description: 'Debusine account used to submit CI builds' @@ -171,8 +163,6 @@ jobs: debian-revision: ${{ inputs.debian-revision }} localversion: ${{ inputs.localversion }} kver-extra: ${{ inputs.kver-extra }} - qcom-next-pr: ${{ inputs.qcom-next-pr }} - kernel-topics-pr: ${{ inputs.kernel-topics-pr }} # Only needed to ask the target workspace what it already has, so it is # checked out only when there is a target workspace to ask. @@ -371,23 +361,16 @@ jobs: # is the one the build above produced, so an archive never carries a version # that was not built and published by this run. # - # The if: is the whole of the decision. A caller that named no workspace - # promotes nowhere, and a run carrying any of the validation overrides - # promotes nowhere either: those builds are deliberately not the kernel the - # matrix describes, and an archive must only ever receive that one. Guarding - # it here rather than in the caller means it holds for every caller, - # including a future one that forwards target-workspace without thinking - # about the overrides. + # The if: is the whole of the decision: a caller that named no workspace + # promotes nowhere. pr-build.yml is that caller, so a pull request's kernel + # never reaches an archive. # --------------------------------------------------------------------------- promote: name: Promote to ${{ inputs.target-workspace }} # build alone: it already needs prepare, so a run prepare skipped as # already published skips the build and this job with it. needs: build - if: >- - ${{ inputs.target-workspace != '' - && inputs.qcom-next-pr == '' - && inputs.kernel-topics-pr == '' }} + if: ${{ inputs.target-workspace != '' }} runs-on: ubuntu-latest permissions: contents: read # actions/checkout diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index eb972300..5a565458 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -81,14 +81,6 @@ on: description: 'Extra suffix appended to the package version' type: string default: '' - qcom-next-pr: - description: 'qcom-next PR numbers to merge (e.g. "42 43")' - type: string - default: '' - kernel-topics-pr: - description: 'kernel-topics PR numbers to apply as patches' - type: string - default: '' permissions: contents: read @@ -128,8 +120,6 @@ jobs: debian-revision: ${{ inputs.debian-revision }} localversion: ${{ inputs.localversion }} kver-extra: ${{ inputs.kver-extra }} - qcom-next-pr: ${{ inputs.qcom-next-pr }} - kernel-topics-pr: ${{ inputs.kernel-topics-pr }} # --------------------------------------------------------------------------- # build: build the prepared source in a suite-matched container, then publish. diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 58215300..4318c967 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -36,20 +36,6 @@ on: required: true default: 'all' type: string - # The matrix says nothing about these: they are one-off validation - # overrides, so they are inputs rather than entry fields. They apply to - # every selected build, and a scheduled run leaves them at their - # defaults. - qcom-next-pr: - description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' - required: false - default: '' - type: string - kernel-topics-pr: - description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' - required: false - default: '' - type: string permissions: contents: read @@ -155,15 +141,8 @@ jobs: kver-extra: ${{ matrix.kver_extra || '' }} debusine-parent-workspace: ${{ matrix.debusine_parent_workspace || '' }} # An entry naming a workspace is promoted into it once its build - # succeeds; one that names none is built and published to S3 only. The - # entry is forwarded as written: build-kernel-debian.yml is what declines - # to promote a run carrying the validation overrides below, so a build - # that is not the plain nightly cannot reach an archive by any caller. + # succeeds; one that names none is built and published to S3 only. target-workspace: ${{ matrix.target_workspace || '' }} - # Dispatch-only validation overrides. A scheduled run carries no inputs, - # so each falls back to the off value and builds the matrix as written. - qcom-next-pr: ${{ inputs.qcom-next-pr || '' }} - kernel-topics-pr: ${{ inputs.kernel-topics-pr || '' }} secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} @@ -195,5 +174,3 @@ jobs: debian-revision: ${{ matrix.debian_revision }} localversion: ${{ matrix.localversion || '' }} kver-extra: ${{ matrix.kver_extra || '' }} - qcom-next-pr: ${{ inputs.qcom-next-pr || '' }} - kernel-topics-pr: ${{ inputs.kernel-topics-pr || '' }} diff --git a/README.md b/README.md index 88a87605..169d37fe 100644 --- a/README.md +++ b/README.md @@ -160,11 +160,11 @@ version this run would produce, and skips the build, the S3 publication and the promotion when it does. A run that skips this way is green: nothing was wrong, there was simply nothing new upstream. -Three kinds of run deliberately promote nowhere: a PR build, which never -forwards a `target_workspace`; a dispatch setting either of the advanced -validation overrides (`qcom-next-pr`, `kernel-topics-pr`), whose kernel is by -definition not the one the matrix describes; and any entry that names no -workspace. +Two kinds of run promote nowhere: a PR build, which never forwards a +`target_workspace`, and any entry that names none. Everything a `daily` +dispatch can say about a build comes from the matrix entry, so there is no way +to dispatch a build that differs from the nightly one and have it reach an +archive. ### Release @@ -510,21 +510,9 @@ promoting: | `upstream-version` | None, required | The version to release without its Debian revision, e.g. `7.2.0~rc7+20260821`. Each selected entry promotes this plus its own `debian_revision`. | | `release-workspace` | `qli` | The Debusine workspace to promote into. | -`daily` carries two further inputs, which the matrix deliberately says nothing -about because they belong to a one-off validation run rather than to a delivery -target. They apply to every selected build, and a scheduled run leaves them at -their defaults: - -| Input | Default | Purpose | -| --- | --- | --- | -| `qcom-next-pr` | Empty | Advanced Qualcomm-only override: `qcom-next` PR numbers to merge before building. | -| `kernel-topics-pr` | Empty | Advanced Qualcomm-only override: `kernel-topics` PR numbers to apply as patches. | - A `daily` dispatch publishes to S3 and, for an entry that names one, promotes -into that entry's `target_workspace` — unless it sets one of the advanced -overrides above, which suppress promotion because the kernel they build is not -the one the matrix describes. Promotion into the release workspace happens only -through `release.yml`. The build workflows themselves +into that entry's `target_workspace`. Promotion into the release workspace +happens only through `release.yml`. The build workflows themselves (`build-kernel-debian.yml`, `build-kernel-ubuntu.yml`) are `workflow_call` only and cannot be dispatched: one run of each is one matrix entry, and a reusable workflow cannot fan itself out over a list. From 67d8f8948dc6c41c75236d9fa28398370ec48d45 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 38/71] ci: let the run decide where a build is published, not the entry target_workspace was a matrix field, so an entry said both what kernel to build and where the result belongs. Those are not the same kind of fact. What to build is a property of the kernel and belongs in the matrix; where it goes is a property of the run -- the nightly publishes to staging, a pull request publishes nowhere, a release publishes to production -- and the same entry takes all three paths depending only on which workflow picked it up. Keeping the destination in the entry forced the distinction to be encoded twice. The matrix carried it per entry, and every consumer then had to work around it: pr-build.yml had to remember not to forward the field, release.yml had to filter on it to find out which entries were releasable, and the resolver had to reject it on Ubuntu suites because only Debian has a promotion path at all. None of those were saying anything about a kernel. So the field is gone and each caller states its own destination. daily.yml names the staging workspace for its Debian legs, overridable through a DEBUSINE_STAGING_WORKSPACE repository variable. pr-build.yml names nothing, as before, so a pull request's kernel is still built and tested and still reaches no archive. release.yml takes from-workspace and to-workspace, defaulting to qli-staging and qli, and selects with --family debian rather than by looking for a field -- which is the real rule, since promotion runs through Debusine and only Debian is built there. One consequence worth stating: every Debian entry the nightly builds now reaches staging, mainline and next and the topic branches included. They were held out before by naming no workspace. Their packages are separate source packages under their own names, so they coexist there, and the Production gate on release.yml is what still decides that none of them reaches qli. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 10 +- .github/workflows/daily.yml | 24 +++-- .github/workflows/pr-build.yml | 8 +- .github/workflows/release.yml | 58 ++++++------ README.md | 109 ++++++++++------------ ci/build-matrix.yaml | 32 +++---- ci/scripts/resolve-matrix.py | 23 +---- 7 files changed, 116 insertions(+), 148 deletions(-) diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 71f5774c..fbdaf5b5 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -15,8 +15,8 @@ name: build-kernel-debian # per Debian entry it selected. That also keeps every value describing a build # in ci/build-matrix.yaml, rather than in a dispatch form that drifts from it. # -# There is no separate release build. An entry that names a target-workspace is -# promoted into it by this workflow, and release.yml promotes onward from that +# There is no separate release build. A caller that names a target-workspace +# has the result promoted into it, and release.yml promotes onward from that # workspace without building anything, so what is released is the artifact this # build produced and tested rather than a second build of the same ref. # @@ -94,9 +94,9 @@ on: target-workspace: description: > Debusine workspace to promote the built package into, empty to - promote nowhere. Set from the matrix entry's target_workspace by the - nightly build; left empty by pr-build.yml, whose builds must not - reach an archive. + promote nowhere. The caller decides: the nightly build names the + staging workspace, and pr-build.yml leaves it empty because a pull + request's kernel must not reach an archive. type: string default: '' secrets: diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 4318c967..c1667e98 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -9,18 +9,19 @@ name: daily # a new one needs no workflow edit. # # This is the only workflow that builds a kernel on a schedule, and every -# package the archives carry comes from a run of it: an entry naming a -# target_workspace is promoted into that workspace as part of its build. -# release.yml promotes onward from there and builds nothing, so nothing is -# ever released that this workflow has not built and tested. +# package the archives carry comes from a run of it: its Debian entries are +# promoted into the staging workspace as part of the build. release.yml +# promotes onward from there and builds nothing, so nothing is ever released +# that this workflow has not built and tested. # # Matrix schema: ci/build-matrix.yaml # One entry is one generated package, so ci/scripts/resolve-matrix.py passes # the selected entries through as they stand. Every field (name, suite, # flavour, srcpkg, binpkg, kernel_config, dkms, debian_revision, git_clone, -# branch_or_tag, ref_strategy, target_workspace) becomes an input of the -# build workflow for the entry's family. name labels the job; flavour is what -# reaches the built kernel's version and its published paths. +# branch_or_tag, ref_strategy) becomes an input of the build workflow for the +# entry's family. name labels the job; flavour is what reaches the built +# kernel's version and its published paths. Where the result is published is +# this workflow's to decide, not the entry's. # # To add a daily build target: add an entry to ci/build-matrix.yaml. No # workflow changes needed. @@ -140,9 +141,12 @@ jobs: localversion: ${{ matrix.localversion || '' }} kver-extra: ${{ matrix.kver_extra || '' }} debusine-parent-workspace: ${{ matrix.debusine_parent_workspace || '' }} - # An entry naming a workspace is promoted into it once its build - # succeeds; one that names none is built and published to S3 only. - target-workspace: ${{ matrix.target_workspace || '' }} + # Every Debian entry the nightly builds is promoted into the staging + # workspace once its build succeeds. The destination belongs to the run + # rather than to the entry: this is the nightly, so its results are what + # staging is for. A PR build of the same entry passes nothing here and + # promotes nowhere, and release.yml promotes onward from staging. + target-workspace: ${{ vars.DEBUSINE_STAGING_WORKSPACE || 'qli-staging' }} secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 3ef53b0a..5eadb602 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -65,10 +65,10 @@ jobs: # fail-fast is disabled so a broken trixie leg still leaves forky and # resolute results on the PR. # - # target-workspace is deliberately not passed on. An entry carrying one is - # promoted into it by the nightly build; a PR build of the same entry only - # has to prove it builds, and must not put a pull request's kernel into an - # archive anyone installs from. + # target-workspace is deliberately not passed on. The nightly build names + # the staging workspace and its results are promoted there; a PR build of the + # same entry only has to prove it builds, and must not put a pull request's + # kernel into an archive anyone installs from. # --------------------------------------------------------------------------- build-debian: name: ${{ matrix.name }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad1cb17d..4bfdea8f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,10 +3,9 @@ name: release # Promote already-built kernel packages into the release workspace. # # This workflow builds nothing. Every package it can release was built, tested -# and published by daily.yml, which promotes each entry carrying a -# target_workspace into that workspace as part of the nightly run. A release -# copies one of those versions onward into the release workspace, so what ships -# is the artifact that was tested rather than a second build of the same ref. +# and promoted into the staging workspace by daily.yml. A release copies one of +# those versions onward into the release workspace, so what ships is the +# artifact that was tested rather than a second build of the same ref. # # That is why there is no release ref anywhere in ci/build-matrix.yaml, and why # releasing needs no change to it: a release names a version that already @@ -35,7 +34,12 @@ on: description: 'Version to release, without the Debian revision (e.g. 7.2.0~rc7+20260821)' required: true type: string - release-workspace: + from-workspace: + description: 'Debusine workspace to release out of, where the nightly build put the packages' + required: true + default: 'qli-staging' + type: string + to-workspace: description: 'Debusine workspace to release into' required: true default: 'qli' @@ -49,11 +53,9 @@ jobs: # --------------------------------------------------------------------------- # configure-matrix: select the entries that can be released. # - # An entry is releasable when it names a target_workspace, because that is - # the workspace holding the nightly artifacts to promote from. The entries - # without one -- mainline, next, the topic branches -- are built for early - # warning and published to S3, so there is nothing to release and asking for - # one by name is a mistake worth reporting. + # The Debian entries, because promotion runs through Debusine and only they + # are built there. Naming an Ubuntu build fails the run with the entries that + # can be released, rather than promoting nothing and reporting success. # --------------------------------------------------------------------------- configure-matrix: name: Configure release matrix @@ -76,9 +78,12 @@ jobs: BUILDS="${INPUT_BUILDS:-all}" - args=() + # --family debian and no --allow-empty: a selection that resolves + # to nothing releasable fails here with the names that do, rather + # than starting a run with an empty matrix. + args=(--family debian) if [[ "$BUILDS" == "all" ]]; then - echo "Source: ci/build-matrix.yaml (every releasable entry)" + echo "Source: ci/build-matrix.yaml (every Debian entry)" else echo "Source: ci/build-matrix.yaml (builds: $BUILDS)" # A name matching no entry fails here, so a typo stops the run @@ -86,27 +91,17 @@ jobs: args+=(--build "$BUILDS") fi - # An entry with no target_workspace has no workspace to promote out - # of, so it is not releasable however it was selected. - releasable='[.[] | select(.target_workspace != "" and .target_workspace != null)]' - MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}" | jq -c "$releasable") - - if [[ "$(echo "$MATRIX" | jq length)" -eq 0 ]]; then - # Listed from the whole matrix, not from the failed selection, - # which by definition contains none of them. - ALL=$(ci/scripts/resolve-matrix.py \ - | jq -r "$releasable"' | map(.name) | join(", ")') - echo "::error::None of the selected builds names a target_workspace, so none of them has anything to release." - echo "::error::Releasable builds: $ALL" + MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}") || { + echo "::error::Releasable builds: $(ci/scripts/resolve-matrix.py --family debian | jq -r 'map(.name) | join(", ")')" exit 1 - fi + } echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" echo "Matrix entries: $(echo "$MATRIX" | jq length)" # --------------------------------------------------------------------------- - # promote: one parallel job per entry, copying its version from the workspace - # the nightly build put it in into the release workspace. + # promote: one parallel job per entry, copying its version from the staging + # workspace the nightly build put it in into the release workspace. # # Nothing is built, so there is no prepare job and no self-hosted runner: # lib/release asks Debusine to publish an artifact it already holds. Both @@ -154,10 +149,11 @@ jobs: DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} - # The entry's own target_workspace is the source: it is where the - # nightly build of this entry put the artifact being released. - DEBUSINE_CI_WORKSPACE: ${{ matrix.target_workspace }} - DEBUSINE_TARGET_WORKSPACE: ${{ inputs.release-workspace }} + # Named by the dispatch, not by the entry: the staging workspace is + # where the nightly build puts every Debian entry, so it is a + # property of the pipeline rather than of the kernel being released. + DEBUSINE_CI_WORKSPACE: ${{ inputs.from-workspace }} + DEBUSINE_TARGET_WORKSPACE: ${{ inputs.to-workspace }} SRCPKG_NAME: ${{ matrix.srcpkg }} UPSTREAM_VERSION: ${{ inputs.upstream-version }} DEBIAN_REVISION: ${{ matrix.debian_revision }} diff --git a/README.md b/README.md index 169d37fe..cb0481fd 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,13 @@ kernel flavour owns as many entries as it has suites. ### Configured variants -| Variant | Source package | Image metapackage | Suites | Promoted to | Notes | -|---------|----------------|-------------------|--------|-------------|-------| -| `qcom-next` | `linux-qcom-next` | `linux-image-qcom-next` | trixie, forky, resolute | `qli-staging` (trixie, forky) | Standard kernel | -| `qcom-next-debug` | `linux-qcom-next-debug` | `linux-image-qcom-next-debug` | trixie, forky | `qli-staging` | Adds `arch/arm64/configs/qcom_debug.config` and `kernel/configs/debug.config` from the kernel source, via `intree:` entries | -| `qcom-arduino` | `linux-qcom-arduino` | `linux-image-qcom-arduino` | trixie, forky | none | Arduino hardware-enablement topic branch (`early/hwe/arduino` of `kernel-topics`) | -| `mainline` | `linux-mainline` | `linux-image-mainline` | trixie, forky | none | Tip of Linus's tree, tracked for early warning of upstream breakage. No DKMS modules | -| `next` | `linux-next` | `linux-image-next` | trixie, forky | none | Newest `next-YYYYMMDD` tag of linux-next. No DKMS modules | +| Variant | Source package | Image metapackage | Suites | Notes | +|---------|----------------|-------------------|--------|-------| +| `qcom-next` | `linux-qcom-next` | `linux-image-qcom-next` | trixie, forky, resolute | Standard kernel | +| `qcom-next-debug` | `linux-qcom-next-debug` | `linux-image-qcom-next-debug` | trixie, forky | Adds `arch/arm64/configs/qcom_debug.config` and `kernel/configs/debug.config` from the kernel source, via `intree:` entries | +| `qcom-arduino` | `linux-qcom-arduino` | `linux-image-qcom-arduino` | trixie, forky | Arduino hardware-enablement topic branch (`early/hwe/arduino` of `kernel-topics`) | +| `mainline` | `linux-mainline` | `linux-image-mainline` | trixie, forky | Tip of Linus's tree, tracked for early warning of upstream breakage. No DKMS modules | +| `next` | `linux-next` | `linux-image-next` | trixie, forky | Newest `next-YYYYMMDD` tag of linux-next. No DKMS modules | `derive-localversion.sh` folds the *flavour* into LOCALVERSION, so each produces a distinct kernel release (`+qcom-next--g`, @@ -35,17 +35,16 @@ image package that can be installed alongside the others. The flavour is what the kernel is; a build's `name` is only what CI calls it. See [docs/version.md](docs/version.md) for how the version strings are composed. -The last three name no `target_workspace`: they track a moving upstream for -early warning, are published to S3, and never reach an archive anyone installs -from. +The last three track a moving upstream for early warning. They are built and +promoted like the rest; nothing about an entry says where it goes. `ci/build-matrix.yaml` is the source of truth; this table is a summary. There is one kind of entry, and one workflow that builds one: - **`daily.yml`** builds every entry nightly, using the matrix-selected - latest-tag or branch-tip strategy, and promotes each entry that names a - `target_workspace` into that workspace. + latest-tag or branch-tip strategy, and promotes its Debian entries into the + staging workspace. - **`release.yml`** builds nothing. It promotes a version that is already in a staging workspace onward into the release workspace. @@ -76,7 +75,6 @@ builds: - iris-vpu - audioreach debian_revision: '0qli1~bpo13+1~' - target_workspace: qli-staging ``` Entries are written out in full rather than sharing YAML anchors, so each one @@ -114,7 +112,7 @@ variant is a matrix change, not a workflow redesign. | --- | --- | --- | | `daily.yml` | Resolves and runs the matrix. The manual build entry point. | Scheduled daily at `23:00 UTC`, or manual dispatch. | | `release.yml` | Promotes an already-built version into the release workspace. Builds nothing. | Manual dispatch only. | -| `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine, publishes it to S3, and promotes it into the entry's `target_workspace` if it has one. | Called by Daily and PR build. | +| `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine, publishes it to S3, and promotes it into the workspace its caller named, if any. | Called by Daily and PR build. | | `build-kernel-ubuntu.yml` | Builds one Ubuntu-suite entry on the Docker path and publishes it to S3. | Called by Daily and PR build. | The two build workflows share their steps through two composite actions rather @@ -147,8 +145,8 @@ archive comes from a run of it. the configured branch directly. - Debian suites build in Debusine, then their `.deb` outputs are downloaded and uploaded to the configured S3 bucket. -- An entry naming a `target_workspace` is then promoted into it with Debusine's - `package-publish` workflow, making it installable from that archive. +- Debian entries are then promoted into the staging workspace with Debusine's + `package-publish` workflow, making them installable from that archive. - `resolute` stays on the Docker-based Ubuntu path and uploads its package outputs to the existing temporary-package S3 location. @@ -160,27 +158,28 @@ version this run would produce, and skips the build, the S3 publication and the promotion when it does. A run that skips this way is green: nothing was wrong, there was simply nothing new upstream. -Two kinds of run promote nowhere: a PR build, which never forwards a -`target_workspace`, and any entry that names none. Everything a `daily` -dispatch can say about a build comes from the matrix entry, so there is no way -to dispatch a build that differs from the nightly one and have it reach an -archive. +Where a build is published belongs to the run, not to the entry: the nightly +names the staging workspace, and `pr-build.yml` names nothing, so a pull +request's kernel is built and tested but reaches no archive. Everything a +`daily` dispatch can say about a build comes from the matrix entry, so there is +no way to dispatch a build that differs from the nightly one at all. ### Release Release is the controlled promotion path, and it builds nothing at all. -- It is manual only, and takes three fields: which **Builds** to release (`all` +- It is manual only, and takes four fields: which **Builds** to release (`all` or a comma-separated list, as Daily does), the **Upstream version** to - release, and the **Release workspace** to release into (`qli` by default). -- Each selected entry promotes `-` from - its own `target_workspace` into the release workspace. The upstream half is - the kernel and the date its tag was cut, and is the same across the suites - built from one ref; the revision half is the entry's own, because a suite's - packages are versioned to sort against the other suites. -- Only an entry that names a `target_workspace` can be released. There is - nowhere to promote from otherwise, so selecting only such entries fails the - run with the list of the ones that can be released. + release, the **From workspace** to release out of (`qli-staging` by default, + where the nightly build put the packages), and the **To workspace** to + release into (`qli` by default). +- Each selected entry promotes `-`. The + upstream half is the kernel and the date its tag was cut, and is the same + across the suites built from one ref; the revision half is the entry's own, + because a suite's packages are versioned to sort against the other suites. +- Only the Debian entries can be released, because promotion runs through + Debusine and only they are built there. Naming an Ubuntu build fails the run + with the list of entries that can be released. - Promotion uses Debusine's `package-publish` workflow, the same operation the nightly build performs one step earlier. - The job runs in the **Production** GitHub environment, which provides the @@ -190,10 +189,6 @@ Releasing changes no file in this repository: there is no ref to pin and no entry to add, because the version being released has already been built. The release decision is the dispatch. -Only the Debian family has a promotion path at all, because promotion runs -through Debusine. `resolve-matrix.py` rejects a `target_workspace` on any other -suite rather than letting it build and then silently not promote. - ## Matrix Model `ci/build-matrix.yaml` is a mapping with exactly one top-level key, `builds`. @@ -218,14 +213,12 @@ them to the workflow matrix as they stand. Each entry carries: | `debian_revision` | The Debian revision this package is built at, stated outright. Carried into every archive the package reaches, because a release promotes the built artifact rather than rebuilding it. | | `localversion`, `kver_extra` | Optional version overrides forwarded to packaging. | | `debusine_parent_workspace` | Optional parent workspace override for the variant's CI child workspaces. | -| `target_workspace` | Debusine workspace the nightly build promotes this entry into, and the workspace `release.yml` later promotes it out of. Optional: an entry without one is built and published to S3 and reaches no archive. Debian suites only. | `resolve-matrix.py` rejects the matrix — before any build job starts — where an entry has an unknown field or a missing required one, a malformed variant or suite identifier, an unknown `ref_strategy`, a `tag_pattern` without -`latest_tag`, a `target_workspace` on a non-Debian suite, a `kernel_config` -fragment that escapes the kernel source root or collides with another -fragment's filename, a `dkms` entry that is not a package name stem or repeats, +`latest_tag`, a `kernel_config` fragment that escapes the kernel source root +or collides with another fragment's filename, a `dkms` entry that is not a package name stem or repeats, or a `debian_revision` that is not a valid Debian revision. Because entries are written out in full, the resolver also checks the @@ -287,10 +280,9 @@ This document covers the CI generator. For the packaging internals: `debian/rule targets, the config fragment merge pipeline, DKMS module bundling and the produced package layout see [debian/README.md](debian/README.md). -The family branch below is taken in the caller, when the matrix is resolved, -from the entry's suite. The promotion branch is taken from the entry's own -`target_workspace`. By the time a build workflow starts, there is nothing left -to decide. +Both branches below are taken in the caller. The family follows from the +entry's suite; the promotion follows from which workflow is running. By the +time a build workflow starts, there is nothing left to decide. ```mermaid flowchart LR @@ -300,9 +292,9 @@ flowchart LR R -->|"ubuntu: resolute"| UBU["build-kernel-ubuntu.yml\nbuild-kernel.sh in Docker\nBuild binary packages"] DEB --> S3["Download .deb files\nPublish to S3"] - DEB --> TW{"target_workspace\nset?"} - TW -->|yes| STG["Promote to qli-staging"] - TW -->|no| NONE["No archive"] + DEB --> TW{"Which caller"} + TW -->|"daily.yml"| STG["Promote to qli-staging"] + TW -->|"pr-build.yml"| NONE["No archive"] UBU --> US3["Publish .deb files to S3"] STG -.->|"release.yml, later\nand builds nothing"| QLI["qli\nProduction APT repository"] @@ -325,7 +317,7 @@ flowchart TD B1["configure-matrix\nEntries, split by family"] B2["build-debian legs\nqcom-next · qcom-next-debug · qcom-arduino\nmainline · next / trixie · forky"] B5["build-ubuntu legs\nqcom-next / resolute"] - B3["configure-matrix\nEntries with a target_workspace"] + B3["configure-matrix\nDebian entries"] end subgraph build[One build workflow per leg] @@ -333,7 +325,7 @@ flowchart TD C3["build\ndebusine-build action"] C4["build\nbuild-kernel.sh in Docker"] C5["publish\nDownload .deb files, upload to S3"] - C6["promote\nlib/release into target_workspace"] + C6["promote\nlib/release into the caller's workspace"] end subgraph outputs[Outputs] @@ -358,8 +350,8 @@ flowchart TD D3 -.-> B3 ``` -A leg runs every job drawn under it except `promote`, which only entries naming -a `target_workspace` reach. `release.yml` touches none of the build column: it +A leg runs every job drawn under it except `promote`, which only a caller +naming a workspace reaches. `release.yml` touches none of the build column: it reads what is already in `qli-staging`. ### Prepare stage @@ -397,7 +389,7 @@ flowchart LR APT --> S3["S3\npackage artifacts"] end - subgraph promote[promote job: only when target_workspace is set] + subgraph promote[promote job: only when the caller named a workspace] WS --> PROMOTE["lib/release\nStart package-publish"] PROMOTE --> STG["qli-staging\nDebusine APT repository"] end @@ -491,7 +483,7 @@ Both name what to act on in one **Builds** field. | Input | Default | Purpose | | --- | --- | --- | -| `builds` | `all` | `all` selects every entry — for `release`, every entry that names a `target_workspace`. Otherwise a comma-separated list of build `name` values from `ci/build-matrix.yaml`, e.g. `qcom-next-trixie,qcom-next-debug-forky`. | +| `builds` | `all` | `all` selects every entry — for `release`, every Debian entry. Otherwise a comma-separated list of build `name` values from `ci/build-matrix.yaml`, e.g. `qcom-next-trixie,qcom-next-debug-forky`. | Everything else about a build — its suite, flavour, kernel repository and ref, package names, config fragments, DKMS modules and Debian revision — comes from @@ -508,11 +500,12 @@ promoting: | Input | Default | Purpose | | --- | --- | --- | | `upstream-version` | None, required | The version to release without its Debian revision, e.g. `7.2.0~rc7+20260821`. Each selected entry promotes this plus its own `debian_revision`. | -| `release-workspace` | `qli` | The Debusine workspace to promote into. | +| `from-workspace` | `qli-staging` | The Debusine workspace to promote out of, where the nightly build put the packages. | +| `to-workspace` | `qli` | The Debusine workspace to promote into. | -A `daily` dispatch publishes to S3 and, for an entry that names one, promotes -into that entry's `target_workspace`. Promotion into the release workspace -happens only through `release.yml`. The build workflows themselves +A `daily` dispatch publishes to S3 and promotes its Debian entries into the +staging workspace, exactly as the scheduled run does. Promotion into the +release workspace happens only through `release.yml`. The build workflows themselves (`build-kernel-debian.yml`, `build-kernel-ubuntu.yml`) are `workflow_call` only and cannot be dispatched: one run of each is one matrix entry, and a reusable workflow cannot fan itself out over a list. @@ -557,9 +550,7 @@ To add a kernel variant: `resolve-matrix.py` rejects the matrix if they drift apart. 3. Use `latest_tag` with a dated tag glob, or `branch_tip`, to track a moving upstream; `pinned_ref` freezes the variant on one ref. -4. Give the variant distinct `srcpkg` and `binpkg` values. Set - `target_workspace` on the Debian entries that should be installable from an - archive, and leave it off the ones built only for early warning. +4. Give the variant distinct `srcpkg` and `binpkg` values. 5. Give each entry a `debian_revision` that sorts where its suite belongs relative to the others and collides with no other entry building the same `srcpkg`. diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index dcdaf503..9696f339 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -29,16 +29,13 @@ # target archive carries: the Debian suites bundle camx and iris-vpu, which # Ubuntu does not package. # -# Every entry is built nightly. An entry that names a target_workspace is also -# promoted into it once the build succeeds, so the archive carries the artifact -# the nightly build tested rather than a rebuild of the same ref. Promoting -# onward from there into a release workspace builds nothing and is driven by -# release.yml, which names the version to promote; nothing here pins a release -# ref. mainline, next and the topic branches name no target_workspace: they are -# early-warning builds, published to S3 and nowhere else. -# -# Only a Debian suite may name a target_workspace. Promotion runs through -# Debusine, which does not build the Ubuntu family. +# Every entry is built nightly. Where a build is published is not stated here: +# the workflow that runs the build decides it, because it depends on why the +# build is running rather than on what is being built. The nightly promotes its +# Debian entries into the staging workspace; a PR build of the same entry +# promotes nowhere; and release.yml promotes onward from staging into the +# release workspace, building nothing. So an entry describes a kernel, and +# never a destination. # # debian_revision is stated outright, not derived. The trailing "~" marks a # version as unreleased, and it is carried into the archive as built: what @@ -67,7 +64,6 @@ builds: - iris-vpu - audioreach debian_revision: '0qli1~bpo13+1~' - target_workspace: qli-staging - name: qcom-next-forky suite: forky @@ -85,7 +81,6 @@ builds: - iris-vpu - audioreach debian_revision: '0qli1~' - target_workspace: qli-staging - name: qcom-next-resolute suite: resolute @@ -124,7 +119,6 @@ builds: - iris-vpu - audioreach debian_revision: '0qli1~bpo13+1~' - target_workspace: qli-staging - name: qcom-next-debug-forky suite: forky @@ -144,13 +138,11 @@ builds: - iris-vpu - audioreach debian_revision: '0qli1~' - target_workspace: qli-staging # --------------------------------------------------------------------------- # qcom-arduino — the Arduino hardware-enablement topic branch. # - # No target_workspace: a topic branch is built for early warning, not for - # anyone to install from an archive. + # A topic branch, built for early warning of what it will take to carry. # --------------------------------------------------------------------------- - name: qcom-arduino-trixie suite: trixie @@ -185,9 +177,8 @@ builds: # --------------------------------------------------------------------------- # mainline — Linus's tree, tracked for early warning of upstream breakage. # - # No target_workspace, and no DKMS: the out-of-tree modules target the - # Qualcomm tree, and a listed module is a presence contract that would fail - # the build here. + # No DKMS: the out-of-tree modules target the Qualcomm tree, and a listed + # module is a presence contract that would fail the build here. # --------------------------------------------------------------------------- - name: mainline-trixie suite: trixie @@ -217,8 +208,7 @@ builds: # next — linux-next, the integration tree, one step ahead of mainline. # # next-YYYYMMDD tags are cut daily, so latest_tag resolves the newest of - # them the same way qcom-next-* does. No target_workspace and no DKMS, as - # above. + # them the same way qcom-next-* does. No DKMS, as above. # --------------------------------------------------------------------------- - name: next-trixie suite: trixie diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index b9a93d47..8350faf7 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -9,11 +9,11 @@ script validates the whole document, selects the entries a caller asked for, and prints them. -There is one kind of entry. Every entry is built nightly, and an entry that -names a target_workspace is also promoted into it, so the archive carries the -artifact the nightly build tested rather than a second build of the same ref. -Promoting that artifact onward to a release workspace is a separate step that -builds nothing; see release.yml. +There is one kind of entry, and it describes a kernel rather than a +destination. Where a build is published is decided by the workflow running it, +not stated here: the nightly promotes its Debian entries into the staging +workspace, a PR build promotes nowhere, and release.yml promotes onward from +staging without building anything. The document is validated in full on every invocation, not just the selected entries, so a typo in an entry nobody selected fails the run that would have @@ -118,7 +118,6 @@ def family_for(suite): OPTIONAL_STRING_FIELDS = ( "tag_pattern", - "target_workspace", "localversion", "kver_extra", "debusine_parent_workspace", @@ -288,18 +287,6 @@ def check_entry(entry, report): elif "tag_pattern" in entry: report("tag_pattern is only valid with ref_strategy=latest_tag") - # Promotion runs through Debusine, and only the Debian family is built - # there. An Ubuntu entry naming a target_workspace would build the package - # and then have no workspace to promote it from, publishing to the S3 path - # and reporting success without the archive ever gaining anything. - if entry.get("target_workspace"): - suite = entry.get("suite") - if isinstance(suite, str) and family_for(suite) != "debian": - report( - f"target_workspace needs a Debian suite, not {suite}; promotion " - "runs through Debusine, which builds " + ", ".join(DEBIAN_SUITES) - ) - revision = entry.get("debian_revision") if isinstance(revision, str) and revision and not REVISION_RE.match(revision): report( From f8dc88b65ac9ffac2b29a853d156759e28a03e9e Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 39/71] ci: resolve the nightly's build-dependencies from released archives only Every Debian build read "qli qli-staging" for its Build-Depends. That was right when a release was its own build from a pinned ref reading qli alone: the daily could resolve against staging because the thing that shipped resolved against released packages and would have failed if a dependency was missing there. Releasing by promotion removes that second build, and with it the check. The artifact a release publishes is now the one the nightly produced, so whatever the nightly resolved a Build-Depends against is what the released kernel depends on. Left alone, a kernel could reach qli needing a -dkms package that only ever reached qli-staging, and nothing in the pipeline would notice until someone tried to install it. So the workspace list becomes an input and each caller states its own. The nightly reads qli, because it is the build a release promotes. pr-build.yml reads qli and qli-staging, because nothing it builds is promoted anywhere and a kernel and the module it needs should be reviewable in one pull request. This is stricter than what the nightly had, so it will fail where the previous setting quietly succeeded: if a -dkms package the matrix lists has not been released to qli, the nightly build of every entry bundling it stops until it is. That failure is the point -- it is the released kernel's dependency that cannot be satisfied -- but it lands on the nightly rather than on a release, so expect it there first. Signed-off-by: Christopher Obbard --- .github/actions/debusine-build/action.yml | 7 +++---- .github/workflows/build-kernel-debian.yml | 13 ++++++++++--- .github/workflows/daily.yml | 5 +++++ .github/workflows/pr-build.yml | 5 +++++ README.md | 6 ++++++ 5 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.github/actions/debusine-build/action.yml b/.github/actions/debusine-build/action.yml index 17b3140a..86a7404e 100644 --- a/.github/actions/debusine-build/action.yml +++ b/.github/actions/debusine-build/action.yml @@ -4,7 +4,7 @@ description: > Debusine, returning the CI workspace it was built in. # Called by build-kernel-debian.yml, which decides what becomes of the result: -# every build is downloaded and uploaded to S3, and one whose entry names a +# every build is downloaded and uploaded to S3, and one whose caller named a # target workspace is also promoted into it. Holding these steps in a composite # action keeps the submission itself in one place, separate from the job that # declares the runner, container and environment it happens in. @@ -41,9 +41,8 @@ inputs: extra-build-dep-workspaces: description: > Workspaces the build resolves its build-dependencies from, space - separated. The nightly build reads qli-staging as well as qli, because a - kernel may need a -dkms package version that has reached staging and not - yet been released. + separated. Set by the calling workflow, which knows whether this build is + one a release may promote. default: 'qli' debusine-action-ref: description: 'Ref of qualcomm-linux/debusine-action to use' diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index fbdaf5b5..34bf5dca 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -91,6 +91,15 @@ on: description: 'Parent Debusine workspace for CI child workspace creation' type: string default: '' + extra-build-dep-workspaces: + description: > + Workspaces the build resolves its Build-Depends from, space + separated. The caller decides, because it follows from what becomes + of the result: the nightly is what a release promotes, so it reads + released archives only, and a PR build may legitimately need a + package that has reached staging and no further. + type: string + default: 'qli' target-workspace: description: > Debusine workspace to promote the built package into, empty to @@ -282,9 +291,7 @@ jobs: debusine-user: ${{ secrets.DEBUSINE_USER }} debusine-token: ${{ secrets.DEBUSINE_TOKEN }} debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} - # A staging build: also reads qli-staging, which carries package - # versions that have not been released to qli yet. - extra-build-dep-workspaces: qli qli-staging + extra-build-dep-workspaces: ${{ inputs.extra-build-dep-workspaces }} # --------------------------------------------------------------------------- # publish: download the .deb files from Debusine and upload them to S3. diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index c1667e98..e23e5848 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -141,6 +141,11 @@ jobs: localversion: ${{ matrix.localversion || '' }} kver-extra: ${{ matrix.kver_extra || '' }} debusine-parent-workspace: ${{ matrix.debusine_parent_workspace || '' }} + # Released archives only. This build is the one a release promotes, so + # anything it resolves a Build-Depends against is something the released + # kernel will depend on: reading qli-staging here would let a kernel + # reach qli depending on a -dkms package that has not. + extra-build-dep-workspaces: qli # Every Debian entry the nightly builds is promoted into the staging # workspace once its build succeeds. The destination belongs to the run # rather than to the entry: this is the nightly, so its results are what diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 5eadb602..89f14be7 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -99,6 +99,11 @@ jobs: # Mark the version so PR artifacts never collide with the daily build's # in the shared Debusine workspace. kver-extra: -pr${{ github.event.number }} + # Staging as well as released: nothing here is promoted anywhere, so a + # pull request may depend on a -dkms package that has reached qli-staging + # and not yet qli -- which is how a kernel and the module it needs get + # reviewed together. + extra-build-dep-workspaces: qli qli-staging secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} diff --git a/README.md b/README.md index cb0481fd..7e0f9158 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,12 @@ archive comes from a run of it. uploaded to the configured S3 bucket. - Debian entries are then promoted into the staging workspace with Debusine's `package-publish` workflow, making them installable from that archive. +- Debian builds resolve their Build-Depends against `qli` alone. The nightly is + the build a release promotes, so anything it builds against is something the + released kernel will depend on; reading `qli-staging` here would let a kernel + reach `qli` depending on a `-dkms` package that has not. A PR build resolves + against `qli qli-staging`, because nothing it produces is promoted and a + kernel and the module it needs should be reviewable together. - `resolute` stays on the Docker-based Ubuntu path and uploads its package outputs to the existing temporary-package S3 location. From 7a740b4079a016bfd23bd307ac8644315b14aea7 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 40/71] ci: fold the release promotion into the nightly build release.yml promoted out of qli-staging into qli, which meant the pipeline needed qli-staging to exist at all: a workflow that runs days after the build has to read the packages from somewhere durable, and the workspace the build actually ran in is not that. lib/build names it -gh---- and creates it per run, so by the time anyone dispatches a release there is neither a name to look up nor, eventually, a workspace behind it. qli-staging was there to outlive the run. Promoting inside the build removes the need for it. build-kernel-debian.yml already had the promote job, reading the CI workspace while the run still holds it, so there was never anything for a second workflow to do that this one could not do first. release.yml is deleted, and with it the upstream-version field that existed only so a later run could name a version it had not built. The destination stays qli-staging for now. Nothing about where packages land changes with this commit: the same workspace receives the same artifacts from the same job, and only the workflow that could later move them onward is gone. Pointing DEBUSINE_STAGING_WORKSPACE at qli would publish every night straight into the released archive, so that is a decision to take separately, together with moving the promote job to the Production environment so an approval still stands between a nightly build and qli. README says so where the variable is documented. Nothing reads DEBUSINE_RELEASE_TOKEN any more, and the Production environment is unused. Both are left configured rather than removed, because they are what a gated promotion into qli will want. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 18 ++- .github/workflows/daily.yml | 26 ++-- .github/workflows/release.yml | 171 ---------------------- README.md | 165 +++++++++------------ ci/build-matrix.yaml | 12 +- ci/scripts/resolve-matrix.py | 12 +- 6 files changed, 108 insertions(+), 296 deletions(-) delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 34bf5dca..934153c7 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -15,10 +15,11 @@ name: build-kernel-debian # per Debian entry it selected. That also keeps every value describing a build # in ci/build-matrix.yaml, rather than in a dispatch form that drifts from it. # -# There is no separate release build. A caller that names a target-workspace -# has the result promoted into it, and release.yml promotes onward from that -# workspace without building anything, so what is released is the artifact this -# build produced and tested rather than a second build of the same ref. +# Publishing happens here and nowhere else. A caller that names a +# target-workspace has this run's own build promoted into it, out of the +# ephemeral CI workspace it was built in, so the archive carries the artifact +# this run produced and tested. There is no later step that could publish +# something else, and nothing rebuilds a ref that has already been built. # # Jobs: # prepare (self-hosted): clone, patch, inject debian/, generate the source, @@ -364,9 +365,12 @@ jobs: # promote: copy the built package from the ephemeral CI workspace into the # workspace the caller named, making it installable from that archive. # - # This is what replaces a separate release build. The artifact promoted here - # is the one the build above produced, so an archive never carries a version - # that was not built and published by this run. + # Promotion is in-run by necessity, not by preference: lib/build names the + # CI workspace after the run that created it + # (-gh----) and it does not outlive it, so + # the only moment its contents can be published is while the run still holds + # it. That is why there is no separate release workflow to promote later -- + # there would be nothing left to promote from. # # The if: is the whole of the decision: a caller that named no workspace # promotes nowhere. pr-build.yml is that caller, so a pull request's kernel diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index e23e5848..8c0989ac 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -8,11 +8,11 @@ name: daily # or a comma-separated list of build names. Build names are matrix values, so # a new one needs no workflow edit. # -# This is the only workflow that builds a kernel on a schedule, and every -# package the archives carry comes from a run of it: its Debian entries are -# promoted into the staging workspace as part of the build. release.yml -# promotes onward from there and builds nothing, so nothing is ever released -# that this workflow has not built and tested. +# This is the only workflow that publishes a kernel package. Its Debian +# entries are promoted into the target workspace as part of the build that +# produced them, so what the archive carries is always something this workflow +# built and tested. Nothing promotes afterwards: the CI workspace a build runs +# in is named after that run and does not outlive it. # # Matrix schema: ci/build-matrix.yaml # One entry is one generated package, so ci/scripts/resolve-matrix.py passes @@ -146,11 +146,17 @@ jobs: # kernel will depend on: reading qli-staging here would let a kernel # reach qli depending on a -dkms package that has not. extra-build-dep-workspaces: qli - # Every Debian entry the nightly builds is promoted into the staging - # workspace once its build succeeds. The destination belongs to the run - # rather than to the entry: this is the nightly, so its results are what - # staging is for. A PR build of the same entry passes nothing here and - # promotes nowhere, and release.yml promotes onward from staging. + # Every Debian entry the nightly builds is promoted into this workspace + # once its build succeeds. The destination belongs to the run rather than + # to the entry: this is the nightly, so its results are what the archive + # is for. A PR build of the same entry passes nothing here and promotes + # nowhere. + # + # qli-staging rather than qli: a nightly promotion is unattended, so it + # publishes where an unreviewed kernel belongs. Pointing this at qli -- + # by setting the repository variable -- would put every night's build + # straight into the released archive, so move the promote job to the + # Production environment first, and take its approval gate with it. target-workspace: ${{ vars.DEBUSINE_STAGING_WORKSPACE || 'qli-staging' }} secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 4bfdea8f..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,171 +0,0 @@ -name: release - -# Promote already-built kernel packages into the release workspace. -# -# This workflow builds nothing. Every package it can release was built, tested -# and promoted into the staging workspace by daily.yml. A release copies one of -# those versions onward into the release workspace, so what ships is the -# artifact that was tested rather than a second build of the same ref. -# -# That is why there is no release ref anywhere in ci/build-matrix.yaml, and why -# releasing needs no change to it: a release names a version that already -# exists, and the matrix describes what is built rather than what has been -# blessed. -# -# A dispatch says which builds to release and which version of them. Every -# selected entry promotes -: the -# upstream half is the kernel and the date it was cut, identical across the -# suites built from one ref, and the revision half is the entry's own. Check -# the version against the source workspace listing before dispatching -- a -# version that is not there fails the promotion rather than releasing -# something else. -# -# Triggered manually only, and never on a schedule: a release is a decision. - -on: - workflow_dispatch: - inputs: - builds: - description: 'Builds to release: all, or comma-separated build names from ci/build-matrix.yaml (e.g. qcom-next-trixie,qcom-next-forky)' - required: true - default: 'all' - type: string - upstream-version: - description: 'Version to release, without the Debian revision (e.g. 7.2.0~rc7+20260821)' - required: true - type: string - from-workspace: - description: 'Debusine workspace to release out of, where the nightly build put the packages' - required: true - default: 'qli-staging' - type: string - to-workspace: - description: 'Debusine workspace to release into' - required: true - default: 'qli' - type: string - -permissions: - contents: read - packages: read - -jobs: - # --------------------------------------------------------------------------- - # configure-matrix: select the entries that can be released. - # - # The Debian entries, because promotion runs through Debusine and only they - # are built there. Naming an Ubuntu build fails the run with the entries that - # can be released, rather than promoting nothing and reporting success. - # --------------------------------------------------------------------------- - configure-matrix: - name: Configure release matrix - runs-on: ubuntu-latest - permissions: - contents: read # actions/checkout - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Generate release matrix - id: set-matrix - env: - INPUT_BUILDS: ${{ inputs.builds }} - run: | - set -euo pipefail - - BUILDS="${INPUT_BUILDS:-all}" - - # --family debian and no --allow-empty: a selection that resolves - # to nothing releasable fails here with the names that do, rather - # than starting a run with an empty matrix. - args=(--family debian) - if [[ "$BUILDS" == "all" ]]; then - echo "Source: ci/build-matrix.yaml (every Debian entry)" - else - echo "Source: ci/build-matrix.yaml (builds: $BUILDS)" - # A name matching no entry fails here, so a typo stops the run - # rather than quietly releasing a smaller set than intended. - args+=(--build "$BUILDS") - fi - - MATRIX=$(ci/scripts/resolve-matrix.py "${args[@]}") || { - echo "::error::Releasable builds: $(ci/scripts/resolve-matrix.py --family debian | jq -r 'map(.name) | join(", ")')" - exit 1 - } - - echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" - echo "Matrix entries: $(echo "$MATRIX" | jq length)" - - # --------------------------------------------------------------------------- - # promote: one parallel job per entry, copying its version from the staging - # workspace the nightly build put it in into the release workspace. - # - # Nothing is built, so there is no prepare job and no self-hosted runner: - # lib/release asks Debusine to publish an artifact it already holds. Both - # workspaces are Debusine workspaces, so this is the same operation the - # nightly build performs, one step further along. - # --------------------------------------------------------------------------- - promote: - name: ${{ matrix.name }} - needs: configure-matrix - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} - runs-on: ubuntu-latest - permissions: - contents: read # actions/checkout - packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder - # Production carries the release credential and the approval gate: a - # promotion into the release workspace is the one step here that a person - # has to agree to. - environment: Production - container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie - options: --user 0:0 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - defaults: - run: - shell: bash - steps: - - name: Checkout debusine-action helpers - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - repository: qualcomm-linux/debusine-action - ref: main - path: debusine-action - fetch-depth: 1 - sparse-checkout: | - lib - - - name: Promote to the release workspace - env: - DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} - DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} - DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} - # Named by the dispatch, not by the entry: the staging workspace is - # where the nightly build puts every Debian entry, so it is a - # property of the pipeline rather than of the kernel being released. - DEBUSINE_CI_WORKSPACE: ${{ inputs.from-workspace }} - DEBUSINE_TARGET_WORKSPACE: ${{ inputs.to-workspace }} - SRCPKG_NAME: ${{ matrix.srcpkg }} - UPSTREAM_VERSION: ${{ inputs.upstream-version }} - DEBIAN_REVISION: ${{ matrix.debian_revision }} - SUITE: ${{ matrix.suite }} - run: | - set -eu - # The revision is the entry's, not the dispatcher's: a suite's - # packages are versioned to sort against the other suites, so the - # full version differs per entry even though the kernel does not. - export SRCPKG_VERSION="${UPSTREAM_VERSION}-${DEBIAN_REVISION}" - echo "Releasing $SRCPKG_NAME $SRCPKG_VERSION" \ - "from $DEBUSINE_CI_WORKSPACE to $DEBUSINE_TARGET_WORKSPACE" - debusine-action/lib/release - echo "Released $SRCPKG_NAME $SRCPKG_VERSION to $DEBUSINE_TARGET_WORKSPACE" \ - >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 7e0f9158..a146b43b 100644 --- a/README.md +++ b/README.md @@ -44,13 +44,13 @@ There is one kind of entry, and one workflow that builds one: - **`daily.yml`** builds every entry nightly, using the matrix-selected latest-tag or branch-tip strategy, and promotes its Debian entries into the - staging workspace. -- **`release.yml`** builds nothing. It promotes a version that is already in a - staging workspace onward into the release workspace. + archive as part of the build that produced them. -Releasing therefore changes no file in this repository. The matrix describes -what is built; which build has been blessed is a decision made at dispatch -time, against versions that already exist. +There is no second workflow that publishes later. The Debusine workspace a +build runs in is named after that run and does not outlive it, so the only +moment its contents can be published is while the run still holds it — +promotion is in the build or it is nowhere. What the archive carries is +therefore always an artifact some nightly run built and tested. One entry in `builds` is one generated package: a single `name` for a single `suite`. The `name` labels that one build and nothing else — it is the Actions @@ -91,11 +91,10 @@ Each entry states its `debian_revision` outright. The configured values are: | Resolute | `0qli~26.04.1~` | The trailing `~` marks the version as one nobody has blessed yet, and it stays -on the version through release. That is a consequence of releasing by -promotion: what reaches `qli` is the artifact `qli-staging` holds, byte for -byte, so its version is the version it was built with. A release that dropped -the `~` would have to be a different build, which is exactly what this design -removes. +on it in the archive. That is a consequence of publishing by promotion: what +`qli-staging` holds is the artifact the build produced, byte for byte, so its +version is the version it was built with. Dropping the `~` would take a second +build at a different version, which is exactly what this design removes. Ordering across *different* suites depends entirely on the configured revisions: with the values above, Resolute < Trixie < Forky, matching a @@ -111,7 +110,6 @@ variant is a matrix change, not a workflow redesign. | Workflow | Purpose | Trigger | | --- | --- | --- | | `daily.yml` | Resolves and runs the matrix. The manual build entry point. | Scheduled daily at `23:00 UTC`, or manual dispatch. | -| `release.yml` | Promotes an already-built version into the release workspace. Builds nothing. | Manual dispatch only. | | `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine, publishes it to S3, and promotes it into the workspace its caller named, if any. | Called by Daily and PR build. | | `build-kernel-ubuntu.yml` | Builds one Ubuntu-suite entry on the Docker path and publishes it to S3. | Called by Daily and PR build. | @@ -147,12 +145,13 @@ archive comes from a run of it. uploaded to the configured S3 bucket. - Debian entries are then promoted into the staging workspace with Debusine's `package-publish` workflow, making them installable from that archive. -- Debian builds resolve their Build-Depends against `qli` alone. The nightly is - the build a release promotes, so anything it builds against is something the - released kernel will depend on; reading `qli-staging` here would let a kernel - reach `qli` depending on a `-dkms` package that has not. A PR build resolves - against `qli qli-staging`, because nothing it produces is promoted and a - kernel and the module it needs should be reviewable together. +- Debian builds resolve their Build-Depends against `qli` alone. The nightly + is the build whose output is published, so anything it builds against is + something the published kernel depends on; reading `qli-staging` here would + let a kernel be published depending on a `-dkms` package that has not been + released. A PR build resolves against `qli qli-staging`, because nothing it + produces is promoted and a kernel and the module it needs should be + reviewable together. - `resolute` stays on the Docker-based Ubuntu path and uploads its package outputs to the existing temporary-package S3 location. @@ -170,30 +169,38 @@ request's kernel is built and tested but reaches no archive. Everything a `daily` dispatch can say about a build comes from the matrix entry, so there is no way to dispatch a build that differs from the nightly one at all. -### Release - -Release is the controlled promotion path, and it builds nothing at all. - -- It is manual only, and takes four fields: which **Builds** to release (`all` - or a comma-separated list, as Daily does), the **Upstream version** to - release, the **From workspace** to release out of (`qli-staging` by default, - where the nightly build put the packages), and the **To workspace** to - release into (`qli` by default). -- Each selected entry promotes `-`. The - upstream half is the kernel and the date its tag was cut, and is the same - across the suites built from one ref; the revision half is the entry's own, - because a suite's packages are versioned to sort against the other suites. -- Only the Debian entries can be released, because promotion runs through - Debusine and only they are built there. Naming an Ubuntu build fails the run - with the list of entries that can be released. -- Promotion uses Debusine's `package-publish` workflow, the same operation the - nightly build performs one step earlier. -- The job runs in the **Production** GitHub environment, which provides the - release credential and enforces the approval gate. - -Releasing changes no file in this repository: there is no ref to pin and no -entry to add, because the version being released has already been built. The -release decision is the dispatch. +### Publishing + +Promotion happens inside the build, in the `promote` job of +`build-kernel-debian.yml`, using Debusine's `package-publish` workflow. + +- It reads the ephemeral CI child workspace the build ran in and publishes the + source and binary artifacts into the workspace the caller named. +- It is in-run by necessity. `lib/build` names that workspace + `-gh----`, creating it fresh for the + run and not keeping it afterwards, so a later workflow would have neither the + name nor the contents to promote. Publishing is part of the build or it does + not happen. +- Only the Debian family reaches it, because promotion runs through Debusine + and the Ubuntu path does not build there. +- It runs in the **Staging** GitHub environment, unattended, because + `qli-staging` is where an unreviewed nightly kernel belongs. + +Publishing changes no file in this repository: there is no ref to pin and no +entry to bless, because what is published is what was built. + +#### Moving to `qli` + +`qli-staging` is the current destination, set by `DEBUSINE_STAGING_WORKSPACE` +and defaulted in [daily.yml](.github/workflows/daily.yml). Pointing that +variable at `qli` would publish every night's kernel straight into the released +archive with nothing in between, so it is not a change to make on its own. +Moving the `promote` job to the **Production** environment first is what +restores an approval gate — at the cost of every nightly run stopping to wait +for one. A pipeline that wants both unattended nightlies and a gated `qli` +needs the two archives it has today, and a promotion step between them that can +run later; that step in turn needs a durable source, which is what +`qli-staging` is for. ## Matrix Model @@ -216,7 +223,7 @@ them to the workflow matrix as they stand. Each entry carries: | `binpkg` | Kernel image metapackage name. | | `kernel_config` | Extra fragments applied on top of `debian/config-available/`, all of which is applied to every build, one per list element. A bare name selects `debian/config-available/.config`; an `intree:` entry names a fragment shipped by the kernel source, as a path relative to the kernel source root (e.g. `intree:arch/arm64/configs/qcom_debug.config`), so it stays versioned with the kernel it targets. Empty for variants that need nothing beyond `config-available/`; today it carries only `intree:` fragments. `resolve-matrix.py` joins it into the comma-separated `kernel-config` workflow input. | | `dkms` | Out-of-tree DKMS modules built and bundled into the image package, one per list element, each named without the `-dkms` suffix (e.g. `kgsl`). Each needs a `-dkms` package in the suite being built for, so this varies between suites. An empty list bundles nothing. A listed module is a presence contract: a build fails rather than shipping an image without it. `resolve-matrix.py` joins it into the comma-separated `dkms` workflow input, which reaches `prepare-source.sh --dkms`; see [debian/README.md](debian/README.md) for what the packaging does with it. | -| `debian_revision` | The Debian revision this package is built at, stated outright. Carried into every archive the package reaches, because a release promotes the built artifact rather than rebuilding it. | +| `debian_revision` | The Debian revision this package is built at, stated outright. Carried into the archive as built, because publishing promotes the artifact rather than rebuilding it. | | `localversion`, `kver_extra` | Optional version overrides forwarded to packaging. | | `debusine_parent_workspace` | Optional parent workspace override for the variant's CI child workspaces. | @@ -303,7 +310,6 @@ flowchart LR TW -->|"pr-build.yml"| NONE["No archive"] UBU --> US3["Publish .deb files to S3"] - STG -.->|"release.yml, later\nand builds nothing"| QLI["qli\nProduction APT repository"] ``` ## For CI Maintainers @@ -316,14 +322,12 @@ flowchart TD A1["daily.yml\nScheduled full matrix"] A2["daily.yml\nManual: all or named builds"] A5["pr-build.yml\nFull matrix on every PR"] - A3["release.yml\nManual: version to release"] end subgraph matrix[Matrix entry points] B1["configure-matrix\nEntries, split by family"] B2["build-debian legs\nqcom-next · qcom-next-debug · qcom-arduino\nmainline · next / trixie · forky"] B5["build-ubuntu legs\nqcom-next / resolute"] - B3["configure-matrix\nDebian entries"] end subgraph build[One build workflow per leg] @@ -337,7 +341,6 @@ flowchart TD subgraph outputs[Outputs] D1["S3 artifacts"] D3["qli-staging APT repository"] - D2["qli APT repository"] end A1 --> B1 @@ -351,14 +354,10 @@ flowchart TD C4 --> D1 C5 --> D1 C6 --> D3 - A3 --> B3 - B3 -->|"promote only, no build"| D2 - D3 -.-> B3 ``` A leg runs every job drawn under it except `promote`, which only a caller -naming a workspace reaches. `release.yml` touches none of the build column: it -reads what is already in `qli-staging`. +naming a workspace reaches — so a PR build stops at S3. ### Prepare stage @@ -401,21 +400,6 @@ flowchart LR end ``` -### Release path - -Nothing is built. `release.yml` promotes a version that the nightly build -already put in `qli-staging`. - -```mermaid -flowchart LR - STG["qli-staging\nsource and binary artifacts"] --> PROMOTE - - subgraph release[promote job: Production GitHub environment] - PROMOTE["lib/release\nSRCPKG_VERSION = upstream-version + entry revision\nStart package-publish"] - PROMOTE --> QLI["qli\nProduction Debusine APT repository"] - end -``` - ### Ubuntu path ```mermaid @@ -483,13 +467,12 @@ out-of-tree module builds are required. ## Manual Builds -Use **Actions** → **daily** → **Run workflow** for a one-off build, and -**Actions** → **release** to promote a version that has already been built. -Both name what to act on in one **Builds** field. +Use **Actions** → **daily** → **Run workflow** for a one-off build. It names +what to build in one **Builds** field, and takes nothing else. | Input | Default | Purpose | | --- | --- | --- | -| `builds` | `all` | `all` selects every entry — for `release`, every Debian entry. Otherwise a comma-separated list of build `name` values from `ci/build-matrix.yaml`, e.g. `qcom-next-trixie,qcom-next-debug-forky`. | +| `builds` | `all` | `all` selects every entry. Otherwise a comma-separated list of build `name` values from `ci/build-matrix.yaml`, e.g. `qcom-next-trixie,qcom-next-debug-forky`. | Everything else about a build — its suite, flavour, kernel repository and ref, package names, config fragments, DKMS modules and Debian revision — comes from @@ -500,21 +483,12 @@ fails the run with the list of names that do. `daily` routes each selected entry to the workflow that builds its family, so one dispatch can name Debian and Ubuntu builds together. -`release` carries two further inputs, because a promotion has to say what it is -promoting: - -| Input | Default | Purpose | -| --- | --- | --- | -| `upstream-version` | None, required | The version to release without its Debian revision, e.g. `7.2.0~rc7+20260821`. Each selected entry promotes this plus its own `debian_revision`. | -| `from-workspace` | `qli-staging` | The Debusine workspace to promote out of, where the nightly build put the packages. | -| `to-workspace` | `qli` | The Debusine workspace to promote into. | - -A `daily` dispatch publishes to S3 and promotes its Debian entries into the -staging workspace, exactly as the scheduled run does. Promotion into the -release workspace happens only through `release.yml`. The build workflows themselves -(`build-kernel-debian.yml`, `build-kernel-ubuntu.yml`) are `workflow_call` only -and cannot be dispatched: one run of each is one matrix entry, and a reusable -workflow cannot fan itself out over a list. +A `daily` dispatch is not a lesser run: it publishes to S3 and promotes its +Debian entries into the archive exactly as the scheduled run does, because a +dispatch cannot describe a build that differs from the nightly one. The build +workflows themselves (`build-kernel-debian.yml`, `build-kernel-ubuntu.yml`) are +`workflow_call` only and cannot be dispatched: one run of each is one matrix +entry, and a reusable workflow cannot fan itself out over a list. ## Configuration @@ -526,6 +500,7 @@ workflow cannot fan itself out over a list. | `DEBUSINE_HOST` | Production Debusine host. | | `DEBUSINE_SCOPE` | Debusine scope. | | `DEBUSINE_PARENT_WORKSPACE` | Parent workspace used to create per-run CI child workspaces. | +| `DEBUSINE_STAGING_WORKSPACE` | Workspace the nightly build promotes into. Defaults to `qli-staging`; see [Moving to `qli`](#moving-to-qli) before changing it. | ### Secrets @@ -533,14 +508,16 @@ workflow cannot fan itself out over a list. | --- | --- | --- | | `DEBUSINE_USER` | Repository | User for Debusine archive and signing-key access. | | `DEBUSINE_TOKEN` | Repository | Token for Debusine build and artifact operations, including the nightly promotion into `qli-staging`. | -| `DEBUSINE_RELEASE_TOKEN` | Production environment | Token used only to promote into the release workspace. | The `build` and `promote` jobs of `build-kernel-debian.yml` select the -**Staging** GitHub environment; the promotion job of `release.yml` selects -**Production**. That split is what lets the nightly build promote into -`qli-staging` unattended while a release into `qli` still passes through the -production approval gate. `DEBUSINE_TOKEN` therefore needs write access to -`qli-staging`. +**Staging** GitHub environment, which is what lets the nightly build promote +unattended. `DEBUSINE_TOKEN` therefore needs write access to the workspace +`DEBUSINE_STAGING_WORKSPACE` names. + +No workflow reads a production release credential today, because nothing +publishes to `qli`. `DEBUSINE_RELEASE_TOKEN` and the **Production** environment +are left configured for when something does; see +[Moving to `qli`](#moving-to-qli). ## Maintaining the Matrix @@ -563,8 +540,8 @@ To add a kernel variant: 6. Confirm suite-family routing: Debian suites use Debusine; Ubuntu suites use the Docker path. 7. Run a filtered daily validation for the new variant, then a full daily run. - Nothing further is needed to release it: once its packages are in the - staging workspace, `release.yml` can promote them. + Nothing further is needed to publish it: a successful nightly build of a + Debian entry promotes itself. To add a new suite (for an existing or new variant): diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index 9696f339..9adc4adf 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -32,15 +32,13 @@ # Every entry is built nightly. Where a build is published is not stated here: # the workflow that runs the build decides it, because it depends on why the # build is running rather than on what is being built. The nightly promotes its -# Debian entries into the staging workspace; a PR build of the same entry -# promotes nowhere; and release.yml promotes onward from staging into the -# release workspace, building nothing. So an entry describes a kernel, and -# never a destination. +# Debian entries into the archive; a PR build of the same entry promotes +# nowhere. So an entry describes a kernel, and never a destination. # # debian_revision is stated outright, not derived. The trailing "~" marks a -# version as unreleased, and it is carried into the archive as built: what -# release.yml promotes is the tested artifact, not a rebuild at a tidier -# version. The rest of the revision places the suite where it belongs relative +# version as unreleased, and it is carried into the archive as built: what is +# published is the tested artifact, never a rebuild at a tidier version. The +# rest of the revision places the suite where it belongs relative # to the other suites (see the ordering discussion in README.md). No two # entries may produce the same srcpkg at the same debian_revision. diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index 8350faf7..0a116b21 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -11,9 +11,8 @@ There is one kind of entry, and it describes a kernel rather than a destination. Where a build is published is decided by the workflow running it, -not stated here: the nightly promotes its Debian entries into the staging -workspace, a PR build promotes nowhere, and release.yml promotes onward from -staging without building anything. +not stated here: the nightly promotes its Debian entries into the archive, and +a PR build promotes nowhere. The document is validated in full on every invocation, not just the selected entries, so a typo in an entry nobody selected fails the run that would have @@ -151,10 +150,9 @@ def family_for(suite): NAME_RE = re.compile(r"^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$") -# The dispatch forms of daily.yml and release.yml take one builds field, where -# "all" means every entry and anything else is a list of names. A build -# actually called all would be unreachable through them, so the matrix may not -# define one. +# The dispatch form of daily.yml takes one builds field, where "all" means +# every entry and anything else is a list of names. A build actually called all +# would be unreachable through it, so the matrix may not define one. RESERVED_NAMES = ("all",) # A Debian revision: no hyphen (that would start a new revision component) and From f138dbfb88757b8d9e42150df0ae8438f7406d90 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 41/71] ci: state pr-build's empty target-workspace outright The promote job in build-kernel-debian.yml runs when target-workspace is not empty, and pr-build.yml kept a pull request's kernel out of the archives by not passing the input at all -- relying on its default being empty, with the reason recorded in a comment two dozen lines above the block it applied to. That is a lot of weight for an absence to carry. A PR is unreviewed by definition and its packages are versioned -pr, so promoting one would put a kernel nobody approved into an archive people install from, at a version no later build supersedes. Nothing in the with: block said so; a change to the input's default, or a copy of the block into a workflow that should promote, would have flipped it silently. So the value is written where it takes effect, with the reason next to it. Nothing changes at runtime: the input already defaulted to empty and still does. Signed-off-by: Christopher Obbard --- .github/workflows/pr-build.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 89f14be7..8da28d8f 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -64,11 +64,6 @@ jobs: # # fail-fast is disabled so a broken trixie leg still leaves forky and # resolute results on the PR. - # - # target-workspace is deliberately not passed on. The nightly build names - # the staging workspace and its results are promoted there; a PR build of the - # same entry only has to prove it builds, and must not put a pull request's - # kernel into an archive anyone installs from. # --------------------------------------------------------------------------- build-debian: name: ${{ matrix.name }} @@ -104,6 +99,19 @@ jobs: # and not yet qli -- which is how a kernel and the module it needs get # reviewed together. extra-build-dep-workspaces: qli qli-staging + # Empty, and stated rather than left to the input's default: this is the + # one value keeping a pull request's kernel out of the archives. An empty + # target-workspace is what build-kernel-debian.yml's promote job tests + # for, so a PR is built, tested and published to S3 and goes no further. + # + # A PR is unreviewed by definition, and its packages are versioned + # -pr rather than as the nightly's, so promoting one would put a + # kernel nobody has approved into an archive people install from, under a + # version no release will ever supersede. Left implicit, a future edit to + # the default -- or a copy of this block into a workflow that should + # promote -- would flip that silently; spelled out here, changing it + # means saying so. + target-workspace: '' secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} From d7be6e8881d846bca9d3852412732024117330b2 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 42/71] packaging: drop the trailing ~ from every Debian revision The trailing ~ existed to sort a daily build below the release built from the same ref: two builds of one kernel needed two versions, and ~ is what made the daily the lesser of them. resolve-matrix.py enforced the pairing, requiring a Daily revision to be its Release revision plus a ~. There is no second build any more. A kernel is built once, and the artifact that build produced is what any archive holds, so nothing exists for a ~ to sort against. All it does now is make every published package look permanently provisional -- a version that will never be superseded by the non-~ one it implies is coming. So trixie builds at 0qli1~bpo13+1, forky at 0qli1, resolute at 0qli1~26.04.1. The remaining ~ in the trixie and resolute revisions is the backports convention and stays: it is what sorts a trixie backport below a forky build of the same kernel, and the cross-suite order the README documents is unchanged -- resolute < trixie < forky, verified with dpkg --compare-versions. Upgrades are unaffected. Every new version sorts above the ~-suffixed one it replaces, so a machine tracking qli-staging moves onto the next nightly by itself. The first run after this builds and publishes rather than skipping, because the version it produces is genuinely one the archive has not seen. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 2 +- .github/workflows/build-kernel-ubuntu.yml | 2 +- README.md | 20 ++++++------- ci/build-matrix.yaml | 36 ++++++++++++----------- docs/version.md | 22 +++++++------- 5 files changed, 43 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 934153c7..2a725ec3 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -79,7 +79,7 @@ on: debian-revision: description: 'Debian revision component of the package version' type: string - default: '0qli~' + default: '0qli1' localversion: description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' type: string diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 5a565458..c0c36423 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -72,7 +72,7 @@ on: debian-revision: description: 'Debian revision component of the package version' type: string - default: '0qli~' + default: '0qli1' localversion: description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' type: string diff --git a/README.md b/README.md index a146b43b..94028e78 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ builds: - camx - iris-vpu - audioreach - debian_revision: '0qli1~bpo13+1~' + debian_revision: '0qli1~bpo13+1' ``` Entries are written out in full rather than sharing YAML anchors, so each one @@ -86,15 +86,15 @@ Each entry states its `debian_revision` outright. The configured values are: | Suite | Revision | | --- | --- | -| Trixie | `0qli~bpo13+1~` | -| Forky | `0qli~` | -| Resolute | `0qli~26.04.1~` | - -The trailing `~` marks the version as one nobody has blessed yet, and it stays -on it in the archive. That is a consequence of publishing by promotion: what -`qli-staging` holds is the artifact the build produced, byte for byte, so its -version is the version it was built with. Dropping the `~` would take a second -build at a different version, which is exactly what this design removes. +| Trixie | `0qli1~bpo13+1` | +| Forky | `0qli1` | +| Resolute | `0qli1~26.04.1` | + +None of them carries a trailing `~`. That marker existed to sort a daily build +below the release rebuilt from the same ref, and there is no such rebuild: the +version is decided once, when the package is built, and the artifact any +archive holds is that one. With nothing for it to sort against, a trailing `~` +would only make every published kernel look permanently provisional. Ordering across *different* suites depends entirely on the configured revisions: with the values above, Resolute < Trixie < Forky, matching a diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index 9adc4adf..5d22135f 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -35,12 +35,14 @@ # Debian entries into the archive; a PR build of the same entry promotes # nowhere. So an entry describes a kernel, and never a destination. # -# debian_revision is stated outright, not derived. The trailing "~" marks a -# version as unreleased, and it is carried into the archive as built: what is -# published is the tested artifact, never a rebuild at a tidier version. The -# rest of the revision places the suite where it belongs relative -# to the other suites (see the ordering discussion in README.md). No two -# entries may produce the same srcpkg at the same debian_revision. +# debian_revision is stated outright, not derived. It carries no marker for +# where a build is in its life: the version is decided once, when the package is +# built, and the same artifact is what any archive later holds, so there is no +# second version for it to sort against. What the revision does say is where +# the suite belongs relative to the other suites -- "~bpo13+1" puts a trixie +# backport below a forky build of the same kernel (see the ordering discussion +# in README.md). No two entries may produce the same srcpkg at the same +# debian_revision. builds: # --------------------------------------------------------------------------- @@ -61,7 +63,7 @@ builds: - camx - iris-vpu - audioreach - debian_revision: '0qli1~bpo13+1~' + debian_revision: '0qli1~bpo13+1' - name: qcom-next-forky suite: forky @@ -78,7 +80,7 @@ builds: - camx - iris-vpu - audioreach - debian_revision: '0qli1~' + debian_revision: '0qli1' - name: qcom-next-resolute suite: resolute @@ -93,7 +95,7 @@ builds: # kgsl alone: camx-dkms and iris-vpu-dkms are not packaged for Ubuntu. dkms: - kgsl - debian_revision: '0qli1~26.04.1~' + debian_revision: '0qli1~26.04.1' # --------------------------------------------------------------------------- # qcom-next-debug — the same kernel ref built with the in-tree debug @@ -116,7 +118,7 @@ builds: - camx - iris-vpu - audioreach - debian_revision: '0qli1~bpo13+1~' + debian_revision: '0qli1~bpo13+1' - name: qcom-next-debug-forky suite: forky @@ -135,7 +137,7 @@ builds: - camx - iris-vpu - audioreach - debian_revision: '0qli1~' + debian_revision: '0qli1' # --------------------------------------------------------------------------- # qcom-arduino — the Arduino hardware-enablement topic branch. @@ -155,7 +157,7 @@ builds: - kgsl - camx - iris-vpu - debian_revision: '0qli~bpo13+1~' + debian_revision: '0qli1~bpo13+1' - name: qcom-arduino-forky suite: forky @@ -170,7 +172,7 @@ builds: - kgsl - camx - iris-vpu - debian_revision: '0qli~' + debian_revision: '0qli1' # --------------------------------------------------------------------------- # mainline — Linus's tree, tracked for early warning of upstream breakage. @@ -188,7 +190,7 @@ builds: binpkg: linux-image-mainline kernel_config: [] dkms: [] - debian_revision: '0qli~bpo13+1~' + debian_revision: '0qli1~bpo13+1' - name: mainline-forky suite: forky @@ -200,7 +202,7 @@ builds: binpkg: linux-image-mainline kernel_config: [] dkms: [] - debian_revision: '0qli~' + debian_revision: '0qli1' # --------------------------------------------------------------------------- # next — linux-next, the integration tree, one step ahead of mainline. @@ -219,7 +221,7 @@ builds: binpkg: linux-image-next kernel_config: [] dkms: [] - debian_revision: '0qli~bpo13+1~' + debian_revision: '0qli1~bpo13+1' - name: next-forky suite: forky @@ -232,4 +234,4 @@ builds: binpkg: linux-image-next kernel_config: [] dkms: [] - debian_revision: '0qli~' + debian_revision: '0qli1' diff --git a/docs/version.md b/docs/version.md index f396079d..d27aa782 100644 --- a/docs/version.md +++ b/docs/version.md @@ -99,11 +99,17 @@ The cost of `~` is one misleading reading: the version sorts below the same snapshot without a SHA, as though it preceded it. Nothing occupies that slot, because every snapshot build carries a SHA. -The revision (`0qli1~bpo13+1`) is derived separately, from the version stub, the -suite, and whether the build is a Daily or a Release. See the matrix -documentation in the top-level [README](../README.md#matrix-model); the trailing -digit on the stub is the packaging revision, bumped when the packaging changes -but the kernel snapshot does not. +The revision (`0qli1~bpo13+1`) is not derived. Each matrix entry states its own +outright, and it says two things: where the suite belongs relative to the other +suites, and which packaging built it. The `~bpo13+1` is the backports +convention, sorting a trixie build below a forky build of the same kernel; the +trailing digit on the `0qli` stub is the packaging revision, bumped when the +packaging changes but the kernel snapshot does not. See the matrix +documentation in the top-level [README](../README.md#matrix-model). + +Nothing in the revision marks how far a build has got. A kernel is built once +and the artifact that build produced is what any archive holds, so there is no +second version for a marker to sort against. ## Ordering @@ -112,15 +118,11 @@ The full chain for one suite, in the order dpkg sorts it: ```text 7.2.0~rc7+git20260820.1~g011a82096bee-0qli1~bpo13+1 first tag of the 20th 7.2.0~rc7+git20260820.2~g3f2f3ca1a81e-0qli1~bpo13+1 respin, same day -7.2.0~rc7+git20260821~gabcdef123456-0qli1~bpo13+1~ Daily, next snapshot -7.2.0~rc7+git20260821~gabcdef123456-0qli1~bpo13+1 Release of the same +7.2.0~rc7+git20260821~gabcdef123456-0qli1~bpo13+1 next snapshot 7.2.0~rc7+git20260821~gabcdef123456-0qli2~bpo13+1 packaging rebuild 7.2.0+git20260902~g123456789abc-0qli1~bpo13+1 7.2 final ``` -A Daily sorts below the Release of the same snapshot because its revision ends -in a trailing `~`. - ## Moved tags If an upstream tag is re-cut against a different commit, the SHA changes and so From bdc73286cd95a5042bad8487e07f368245beebf8 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 43/71] ci: say what kver-extra marks in pr-build pr-build.yml claimed that kver-extra marks the version so PR artifacts never collide with the daily build's. It does not: debian/rules appends KVER_EXTRA to KVER and derives the package version separately, so a PR build shares the nightly's source version and differs only in uname -r and the versioned linux-image name. That is enough where it is -- a PR build has its own throwaway workspace and promotes nowhere -- but it is not a version stamp, and the comment should not invite anything to start relying on it as one. Signed-off-by: Christopher Obbard --- .github/workflows/pr-build.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 8da28d8f..186e747a 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -91,8 +91,13 @@ jobs: dkms: ${{ matrix.dkms }} debian-revision: ${{ matrix.debian_revision }} localversion: ${{ matrix.localversion || '' }} - # Mark the version so PR artifacts never collide with the daily build's - # in the shared Debusine workspace. + # Marks the kernel release, not the package version: debian/rules + # appends KVER_EXTRA to KVER and then derives the version separately, so + # this changes uname -r and the versioned linux-image name while the + # source version stays the one the nightly would build. That is enough + # here -- a PR build has its own throwaway workspace and promotes + # nowhere -- but it is not a version stamp, and nothing should start + # relying on it as one. kver-extra: -pr${{ github.event.number }} # Staging as well as released: nothing here is promoted anywhere, so a # pull request may depend on a -dkms package that has reached qli-staging From 707795fdf0475ca3ee3db3bc4712ea2ffa7f94bf Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 44/71] WIP: temporarily disable all builds Signed-off-by: Christopher Obbard --- ci/build-matrix.yaml | 322 +++++++++++++++++++++---------------------- 1 file changed, 161 insertions(+), 161 deletions(-) diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index 5d22135f..1b6e928b 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -48,22 +48,22 @@ builds: # --------------------------------------------------------------------------- # qcom-next — the standard kernel. # --------------------------------------------------------------------------- - - name: qcom-next-trixie - suite: trixie - flavour: qcom-next - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next - ref_strategy: latest_tag - tag_pattern: 'qcom-next-*' - srcpkg: linux-qcom-next - binpkg: linux-image-qcom-next - kernel_config: [] - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1~bpo13+1' + # - name: qcom-next-trixie + # suite: trixie + # flavour: qcom-next + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next + # ref_strategy: latest_tag + # tag_pattern: 'qcom-next-*' + # srcpkg: linux-qcom-next + # binpkg: linux-image-qcom-next + # kernel_config: [] + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1~bpo13+1' - name: qcom-next-forky suite: forky @@ -82,156 +82,156 @@ builds: - audioreach debian_revision: '0qli1' - - name: qcom-next-resolute - suite: resolute - flavour: qcom-next - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next - ref_strategy: latest_tag - tag_pattern: 'qcom-next-*' - srcpkg: linux-qcom-next - binpkg: linux-image-qcom-next - kernel_config: [] - # kgsl alone: camx-dkms and iris-vpu-dkms are not packaged for Ubuntu. - dkms: - - kgsl - debian_revision: '0qli1~26.04.1' + # - name: qcom-next-resolute + # suite: resolute + # flavour: qcom-next + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next + # ref_strategy: latest_tag + # tag_pattern: 'qcom-next-*' + # srcpkg: linux-qcom-next + # binpkg: linux-image-qcom-next + # kernel_config: [] + # # kgsl alone: camx-dkms and iris-vpu-dkms are not packaged for Ubuntu. + # dkms: + # - kgsl + # debian_revision: '0qli1~26.04.1' - # --------------------------------------------------------------------------- - # qcom-next-debug — the same kernel ref built with the in-tree debug - # fragments, packaged separately so it installs alongside qcom-next. - # --------------------------------------------------------------------------- - - name: qcom-next-debug-trixie - suite: trixie - flavour: qcom-next-debug - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next - ref_strategy: latest_tag - tag_pattern: 'qcom-next-*' - srcpkg: linux-qcom-next-debug - binpkg: linux-image-qcom-next-debug - kernel_config: - - intree:arch/arm64/configs/qcom_debug.config - - intree:kernel/configs/debug.config - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1~bpo13+1' + # # --------------------------------------------------------------------------- + # # qcom-next-debug — the same kernel ref built with the in-tree debug + # # fragments, packaged separately so it installs alongside qcom-next. + # # --------------------------------------------------------------------------- + # - name: qcom-next-debug-trixie + # suite: trixie + # flavour: qcom-next-debug + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next + # ref_strategy: latest_tag + # tag_pattern: 'qcom-next-*' + # srcpkg: linux-qcom-next-debug + # binpkg: linux-image-qcom-next-debug + # kernel_config: + # - intree:arch/arm64/configs/qcom_debug.config + # - intree:kernel/configs/debug.config + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1~bpo13+1' - - name: qcom-next-debug-forky - suite: forky - flavour: qcom-next-debug - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next - ref_strategy: latest_tag - tag_pattern: 'qcom-next-*' - srcpkg: linux-qcom-next-debug - binpkg: linux-image-qcom-next-debug - kernel_config: - - intree:arch/arm64/configs/qcom_debug.config - - intree:kernel/configs/debug.config - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1' + # - name: qcom-next-debug-forky + # suite: forky + # flavour: qcom-next-debug + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next + # ref_strategy: latest_tag + # tag_pattern: 'qcom-next-*' + # srcpkg: linux-qcom-next-debug + # binpkg: linux-image-qcom-next-debug + # kernel_config: + # - intree:arch/arm64/configs/qcom_debug.config + # - intree:kernel/configs/debug.config + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1' - # --------------------------------------------------------------------------- - # qcom-arduino — the Arduino hardware-enablement topic branch. - # - # A topic branch, built for early warning of what it will take to carry. - # --------------------------------------------------------------------------- - - name: qcom-arduino-trixie - suite: trixie - flavour: qcom-arduino - git_clone: https://github.com/qualcomm-linux/kernel-topics - branch_or_tag: early/hwe/arduino - ref_strategy: branch_tip - srcpkg: linux-qcom-arduino - binpkg: linux-image-qcom-arduino - kernel_config: [] - dkms: - - kgsl - - camx - - iris-vpu - debian_revision: '0qli1~bpo13+1' + # # --------------------------------------------------------------------------- + # # qcom-arduino — the Arduino hardware-enablement topic branch. + # # + # # A topic branch, built for early warning of what it will take to carry. + # # --------------------------------------------------------------------------- + # - name: qcom-arduino-trixie + # suite: trixie + # flavour: qcom-arduino + # git_clone: https://github.com/qualcomm-linux/kernel-topics + # branch_or_tag: early/hwe/arduino + # ref_strategy: branch_tip + # srcpkg: linux-qcom-arduino + # binpkg: linux-image-qcom-arduino + # kernel_config: [] + # dkms: + # - kgsl + # - camx + # - iris-vpu + # debian_revision: '0qli1~bpo13+1' - - name: qcom-arduino-forky - suite: forky - flavour: qcom-arduino - git_clone: https://github.com/qualcomm-linux/kernel-topics - branch_or_tag: early/hwe/arduino - ref_strategy: branch_tip - srcpkg: linux-qcom-arduino - binpkg: linux-image-qcom-arduino - kernel_config: [] - dkms: - - kgsl - - camx - - iris-vpu - debian_revision: '0qli1' + # - name: qcom-arduino-forky + # suite: forky + # flavour: qcom-arduino + # git_clone: https://github.com/qualcomm-linux/kernel-topics + # branch_or_tag: early/hwe/arduino + # ref_strategy: branch_tip + # srcpkg: linux-qcom-arduino + # binpkg: linux-image-qcom-arduino + # kernel_config: [] + # dkms: + # - kgsl + # - camx + # - iris-vpu + # debian_revision: '0qli1' - # --------------------------------------------------------------------------- - # mainline — Linus's tree, tracked for early warning of upstream breakage. - # - # No DKMS: the out-of-tree modules target the Qualcomm tree, and a listed - # module is a presence contract that would fail the build here. - # --------------------------------------------------------------------------- - - name: mainline-trixie - suite: trixie - flavour: mainline - git_clone: https://github.com/torvalds/linux - branch_or_tag: master - ref_strategy: branch_tip - srcpkg: linux-mainline - binpkg: linux-image-mainline - kernel_config: [] - dkms: [] - debian_revision: '0qli1~bpo13+1' + # # --------------------------------------------------------------------------- + # # mainline — Linus's tree, tracked for early warning of upstream breakage. + # # + # # No DKMS: the out-of-tree modules target the Qualcomm tree, and a listed + # # module is a presence contract that would fail the build here. + # # --------------------------------------------------------------------------- + # - name: mainline-trixie + # suite: trixie + # flavour: mainline + # git_clone: https://github.com/torvalds/linux + # branch_or_tag: master + # ref_strategy: branch_tip + # srcpkg: linux-mainline + # binpkg: linux-image-mainline + # kernel_config: [] + # dkms: [] + # debian_revision: '0qli1~bpo13+1' - - name: mainline-forky - suite: forky - flavour: mainline - git_clone: https://github.com/torvalds/linux - branch_or_tag: master - ref_strategy: branch_tip - srcpkg: linux-mainline - binpkg: linux-image-mainline - kernel_config: [] - dkms: [] - debian_revision: '0qli1' + # - name: mainline-forky + # suite: forky + # flavour: mainline + # git_clone: https://github.com/torvalds/linux + # branch_or_tag: master + # ref_strategy: branch_tip + # srcpkg: linux-mainline + # binpkg: linux-image-mainline + # kernel_config: [] + # dkms: [] + # debian_revision: '0qli1' - # --------------------------------------------------------------------------- - # next — linux-next, the integration tree, one step ahead of mainline. - # - # next-YYYYMMDD tags are cut daily, so latest_tag resolves the newest of - # them the same way qcom-next-* does. No DKMS, as above. - # --------------------------------------------------------------------------- - - name: next-trixie - suite: trixie - flavour: next - git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git - branch_or_tag: master - ref_strategy: latest_tag - tag_pattern: 'next-*' - srcpkg: linux-next - binpkg: linux-image-next - kernel_config: [] - dkms: [] - debian_revision: '0qli1~bpo13+1' + # # --------------------------------------------------------------------------- + # # next — linux-next, the integration tree, one step ahead of mainline. + # # + # # next-YYYYMMDD tags are cut daily, so latest_tag resolves the newest of + # # them the same way qcom-next-* does. No DKMS, as above. + # # --------------------------------------------------------------------------- + # - name: next-trixie + # suite: trixie + # flavour: next + # git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git + # branch_or_tag: master + # ref_strategy: latest_tag + # tag_pattern: 'next-*' + # srcpkg: linux-next + # binpkg: linux-image-next + # kernel_config: [] + # dkms: [] + # debian_revision: '0qli1~bpo13+1' - - name: next-forky - suite: forky - flavour: next - git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git - branch_or_tag: master - ref_strategy: latest_tag - tag_pattern: 'next-*' - srcpkg: linux-next - binpkg: linux-image-next - kernel_config: [] - dkms: [] - debian_revision: '0qli1' + # - name: next-forky + # suite: forky + # flavour: next + # git_clone: https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git + # branch_or_tag: master + # ref_strategy: latest_tag + # tag_pattern: 'next-*' + # srcpkg: linux-next + # binpkg: linux-image-next + # kernel_config: [] + # dkms: [] + # debian_revision: '0qli1' From 5f6594c301b3a2bb963f3845ff55ac1ca56788c0 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 45/71] WIP: temporarily allow PR to push into staging Signed-off-by: Christopher Obbard --- .github/workflows/pr-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 186e747a..b629eaf3 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -116,7 +116,8 @@ jobs: # the default -- or a copy of this block into a workflow that should # promote -- would flip that silently; spelled out here, changing it # means saying so. - target-workspace: '' +# target-workspace: '' + target-workspace: qli-staging secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} From a8bafc8cc8594df7d7eca8bafdde4a9e5e04882e Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 46/71] wip: manual suffix Signed-off-by: Christopher Obbard --- ci/build-matrix.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index 1b6e928b..36ebc9d8 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -80,7 +80,8 @@ builds: - camx - iris-vpu - audioreach - debian_revision: '0qli1' + # TODO: added manual revision + debian_revision: '0qli1+staging1' # - name: qcom-next-resolute # suite: resolute From 77a3ae25b724af572eb1ca0d8fc63a81856db156 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 47/71] Revert "WIP: temporarily allow PR to push into staging" This reverts commit dada06c0209a5f2ba0b4450da87f5af8d6ce919d. Signed-off-by: Christopher Obbard --- .github/workflows/pr-build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index b629eaf3..186e747a 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -116,8 +116,7 @@ jobs: # the default -- or a copy of this block into a workflow that should # promote -- would flip that silently; spelled out here, changing it # means saying so. -# target-workspace: '' - target-workspace: qli-staging + target-workspace: '' secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} From cc60ffafc246412b2d3b9199cde81d5e14503a69 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 48/71] ci: bring back a release workflow, building the ref it ships Releasing has had no workflow since the promotion was folded into the nightly build. That commit removed release.yml because its model could not work: it promoted out of qli-staging days after the fact, which meant the pipeline had to keep a durable staging workspace for it to read, since the workspace a build actually ran in is named after that run and does not outlive it. This one does not promote a previous run's build. It builds the ref it releases, in the same workflow the nightly calls, and promotes that build while the run still holds its workspace -- so what reaches qli is the artifact this run produced and tested, and the awkward dependency on a workspace outliving the run that filled it never arises. What is released is written down rather than typed in. ci/build-matrix.yaml gains a releases list beside builds: same shape, plus a target_workspace, and ref_strategy restricted to pinned_ref so that two dispatches of one entry cannot ship different kernels. Updating a release is a pull request that changes branch_or_tag, which puts the ref that ships under review before the run that ships it -- where an upstream-version dispatch field put it at the mercy of whoever typed it. The refs come from main, which is where releasing last worked: qcom-next and qcom-next-debug, trixie and forky, at qcom-next-7.2-20260826. target_workspace stays rejected on a builds entry. Where a nightly is published follows from why it is running and remains the calling workflow's to decide; a release is the one case where the destination belongs to the entry, because putting one ref into one archive is the whole of what it is. The two lists are validated together however few of them a caller selects, so a broken release entry fails a nightly run rather than lying in wait for whoever next tries to release. Releases are Debian-only, enforced rather than assumed: promotion runs through Debusine and the Ubuntu path has no workspace to promote into, so an Ubuntu entry there would name a destination nothing could deliver it to. Which environment the promote job runs in becomes the caller's, through a new promote-environment input defaulting to Staging. The nightly keeps promoting into staging unattended; a release passes Production, so whatever approval that environment requires stands in front of qli. The run summary lists the refs, revisions and destinations before that gate, so the approval is given against what is shipping rather than against a run number. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 20 ++- .github/workflows/release.yml | 159 ++++++++++++++++++++++ README.md | 147 +++++++++++++++----- ci/build-matrix.yaml | 116 +++++++++++++++- ci/scripts/resolve-matrix.py | 153 ++++++++++++++++----- 5 files changed, 517 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 2a725ec3..739224f6 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -109,6 +109,15 @@ on: request's kernel must not reach an archive. type: string default: '' + promote-environment: + description: > + GitHub environment the promote job runs in, and so which approvals + and which credentials stand between a build and the archive it + publishes into. Defaults to Staging, where the nightly promotes + unattended. release.yml passes Production, because a release reaches + an archive people install from and an unreviewed one must not. + type: string + default: 'Staging' secrets: DEBUSINE_USER: description: 'Debusine account used to submit CI builds' @@ -369,12 +378,15 @@ jobs: # CI workspace after the run that created it # (-gh----) and it does not outlive it, so # the only moment its contents can be published is while the run still holds - # it. That is why there is no separate release workflow to promote later -- - # there would be nothing left to promote from. + # it. That is why release.yml builds the ref it releases rather than + # promoting a nightly afterwards -- by then there would be nothing left to + # promote from. # # The if: is the whole of the decision: a caller that named no workspace # promotes nowhere. pr-build.yml is that caller, so a pull request's kernel - # never reaches an archive. + # never reaches an archive. Which archive, and what stands in front of it, + # is the caller's too: promote-environment carries the approval gate, so a + # release into qli can require one where the nightly into staging does not. # --------------------------------------------------------------------------- promote: name: Promote to ${{ inputs.target-workspace }} @@ -386,7 +398,7 @@ jobs: permissions: contents: read # actions/checkout packages: read # container: ghcr.io/qualcomm-linux/debusine-pkg-builder - environment: Staging + environment: ${{ inputs.promote-environment }} container: image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie options: --user 0:0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..0e04d5c9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,159 @@ +name: release + +# Release kernel packages into the released archive. +# +# A release builds the ref it releases. It cannot promote a nightly instead: +# lib/build names the CI workspace after the run that created it and that +# workspace does not outlive the run, so by the time anyone decides to release, +# there is no longer anything to promote from. Building here means the artifact +# that reaches the archive is the one this run produced and tested, and the +# ref it came from is pinned rather than resolved, so two dispatches of one +# entry release the same kernel. +# +# Every value describing a release lives in ci/build-matrix.yaml under +# "releases": the ref, the suites, the packaging, and the workspace each entry +# publishes into. A dispatch only says which of them to run. Updating a release +# is therefore a pull request that changes branch_or_tag, which is what puts +# the ref that ships under review -- rather than a version typed into this form +# at the moment of releasing. +# +# Debian suites only. Promotion runs through Debusine, and the Ubuntu path +# publishes to S3 with no workspace to promote into; resolve-matrix.py rejects +# a non-Debian release entry, so there is no Ubuntu leg to select here. +# +# Triggered manually only, and never on a schedule: a release is a decision. +# The promote job runs in the Production environment, so whatever approval that +# environment requires stands between this run and the archive. + +on: + workflow_dispatch: + inputs: + builds: + description: 'Releases to run: all, or comma-separated build names from ci/build-matrix.yaml (e.g. qcom-next-trixie,qcom-next-forky)' + required: true + default: 'all' + type: string + +permissions: + contents: read + packages: read + +jobs: + # --------------------------------------------------------------------------- + # configure-matrix: select the entries to release from ci/build-matrix.yaml. + # + # --releases reads the releases list rather than builds. Both are validated + # whichever is selected, so a broken release entry fails here rather than + # part-way through a release. + # + # No --allow-empty and no family split: every release entry is Debian, so a + # selection that matches nothing is a mistake in the request and fails. + # --------------------------------------------------------------------------- + configure-matrix: + name: Configure release matrix + runs-on: ubuntu-latest + permissions: + contents: read # actions/checkout + outputs: + releases: ${{ steps.set-matrix.outputs.releases }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Generate release matrix + id: set-matrix + env: + INPUT_BUILDS: ${{ inputs.builds }} + run: | + set -euo pipefail + + BUILDS="${INPUT_BUILDS:-all}" + + args=() + if [[ "$BUILDS" == "all" ]]; then + echo "Source: ci/build-matrix.yaml releases (every entry)" + else + echo "Source: ci/build-matrix.yaml releases (builds: $BUILDS)" + # A name matching no release entry fails here, so releasing under a + # stale or mistyped name stops the run rather than quietly + # releasing a subset of what was asked for. + args+=(--build "$BUILDS") + fi + + MATRIX=$(ci/scripts/resolve-matrix.py --releases "${args[@]}") + + echo "releases=$MATRIX" >> "$GITHUB_OUTPUT" + echo "Release entries: $(echo "$MATRIX" | jq length)" + + - name: Summarise what this run will release + env: + MATRIX: ${{ steps.set-matrix.outputs.releases }} + run: | + set -euo pipefail + # The refs and destinations in one place, so the approval on the + # Production environment is given against what is actually shipping + # rather than against a run number. + { + echo "### Releasing" + echo + echo "| Build | Suite | Ref | Revision | Into |" + echo "| --- | --- | --- | --- | --- |" + jq -r '.[] | "| \(.name) | \(.suite) | `\(.branch_or_tag)` | `\(.debian_revision)` | \(.target_workspace) |"' <<<"$MATRIX" + } >> "$GITHUB_STEP_SUMMARY" + + # --------------------------------------------------------------------------- + # release: one parallel job per selected entry. + # + # The same workflow the nightly calls, differing only in what it is told: a + # pinned ref instead of a resolved one, and the released workspace instead of + # staging. Nothing about how a kernel is built changes because it is being + # released. + # + # fail-fast is disabled so a failed trixie release does not cancel the forky + # one; each entry is a package of its own and one failing does not make the + # others wrong to ship. + # --------------------------------------------------------------------------- + release: + name: ${{ matrix.name }} + needs: configure-matrix + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.configure-matrix.outputs.releases) }} + permissions: + contents: read # actions/checkout in the called workflow + packages: read # ghcr.io images it pulls + uses: ./.github/workflows/build-kernel-debian.yml + with: + suite: ${{ matrix.suite }} + build: ${{ matrix.name }} + flavour: ${{ matrix.flavour }} + git-clone: ${{ matrix.git_clone }} + kernel-branch: ${{ matrix.branch_or_tag }} + ref-strategy: ${{ matrix.ref_strategy }} + srcpkg: ${{ matrix.srcpkg }} + binpkg: ${{ matrix.binpkg }} + kernel-config: ${{ matrix.kernel_config }} + dkms: ${{ matrix.dkms }} + debian-revision: ${{ matrix.debian_revision }} + localversion: ${{ matrix.localversion || '' }} + kver-extra: ${{ matrix.kver_extra || '' }} + debusine-parent-workspace: ${{ matrix.debusine_parent_workspace || '' }} + # Released archives only, as the nightly does. What this build resolves a + # Build-Depends against is something the released kernel will depend on, + # so reading qli-staging here would let a released kernel require a -dkms + # package that has not been released. + extra-build-dep-workspaces: qli + # The entry's own, and the one field that makes it a release rather than + # a build of the same ref: where the package is published. Stated in the + # matrix rather than defaulted here, so the archive a release reaches is + # reviewable alongside the ref it ships. + target-workspace: ${{ matrix.target_workspace }} + # A release reaches an archive people install from, so the promotion + # waits for whatever approval the Production environment requires. The + # nightly leaves this at Staging and promotes unattended. + promote-environment: Production + secrets: + DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} diff --git a/README.md b/README.md index 94028e78..610717f6 100644 --- a/README.md +++ b/README.md @@ -40,16 +40,19 @@ promoted like the rest; nothing about an entry says where it goes. `ci/build-matrix.yaml` is the source of truth; this table is a summary. -There is one kind of entry, and one workflow that builds one: +Entries come in two lists, each with a workflow that builds them: -- **`daily.yml`** builds every entry nightly, using the matrix-selected +- **`daily.yml`** builds every `builds` entry nightly, using the matrix-selected latest-tag or branch-tip strategy, and promotes its Debian entries into the - archive as part of the build that produced them. + staging archive as part of the build that produced them. +- **`release.yml`** builds a `releases` entry on request, from the immutable ref + that entry pins, and promotes it into `qli` behind an approval. -There is no second workflow that publishes later. The Debusine workspace a +Neither publishes a build some earlier run produced. The Debusine workspace a build runs in is named after that run and does not outlive it, so the only moment its contents can be published is while the run still holds it — -promotion is in the build or it is nowhere. What the archive carries is +promotion is in the build or it is nowhere, and a release therefore builds the +ref it ships rather than promoting a nightly. What the archive carries is therefore always an artifact some nightly run built and tested. One entry in `builds` is one generated package: a single `name` for a single @@ -110,7 +113,8 @@ variant is a matrix change, not a workflow redesign. | Workflow | Purpose | Trigger | | --- | --- | --- | | `daily.yml` | Resolves and runs the matrix. The manual build entry point. | Scheduled daily at `23:00 UTC`, or manual dispatch. | -| `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine, publishes it to S3, and promotes it into the workspace its caller named, if any. | Called by Daily and PR build. | +| `release.yml` | Builds the pinned refs in the matrix's `releases` list and promotes them into the released archive. | Manual dispatch only. | +| `build-kernel-debian.yml` | Builds one Debian-suite entry in Debusine, publishes it to S3, and promotes it into the workspace its caller named, if any. | Called by Daily, Release and PR build. | | `build-kernel-ubuntu.yml` | Builds one Ubuntu-suite entry on the Docker path and publishes it to S3. | Called by Daily and PR build. | The two build workflows share their steps through two composite actions rather @@ -129,8 +133,8 @@ needs and shows no skipped job for a path it did not take. ### Daily -Daily is the only workflow that builds a kernel, and every package in every -archive comes from a run of it. +Daily builds every entry in the matrix, and everything in the staging archive +comes from a run of it. - The scheduled run resolves the full matrix. - A manual run says which entries to build in one **Builds** field: @@ -169,6 +173,32 @@ request's kernel is built and tested but reaches no archive. Everything a `daily` dispatch can say about a build comes from the matrix entry, so there is no way to dispatch a build that differs from the nightly one at all. +### Release + +A release builds the ref it releases, and publishes the result into `qli`. + +- Every value describing a release lives in `ci/build-matrix.yaml` under + `releases`: the pinned ref, the suites, the packaging, and the + `target_workspace` each entry publishes into. A dispatch only says which + entries to run, in the same **Builds** field Daily uses. +- Updating a release is a pull request that changes `branch_or_tag`. That is + what puts the ref that ships under review, rather than a version typed into + a dispatch form at the moment of releasing. +- `ref_strategy` must be `pinned_ref`. `latest_tag` would make two dispatches + of one entry release different kernels, and `branch_tip` would make them + release whatever the branch had reached. +- Debian suites only, and `resolve-matrix.py` rejects anything else: promotion + runs through Debusine, and the Ubuntu path has no workspace to promote into. +- The `promote` job runs in the **Production** environment, so whatever + approval that environment requires stands between the run and the archive. +- Build-Depends resolve against `qli` alone, as the nightly's do, so a released + kernel cannot depend on a `-dkms` package that has not itself been released. + +It rebuilds rather than promoting a nightly because there is nothing left to +promote from: the CI workspace a nightly ran in is named after that run and does +not outlive it. What reaches `qli` is therefore the artifact this run built and +tested, from a ref that cannot have moved since it was reviewed. + ### Publishing Promotion happens inside the build, in the `promote` job of @@ -183,32 +213,41 @@ Promotion happens inside the build, in the `promote` job of not happen. - Only the Debian family reaches it, because promotion runs through Debusine and the Ubuntu path does not build there. -- It runs in the **Staging** GitHub environment, unattended, because - `qli-staging` is where an unreviewed nightly kernel belongs. +- Which GitHub environment it runs in is the caller's, through + `promote-environment`. Daily leaves it at **Staging** and promotes + unattended, because `qli-staging` is where an unreviewed nightly kernel + belongs; Release passes **Production**, so an approval stands in front of + `qli`. -Publishing changes no file in this repository: there is no ref to pin and no -entry to bless, because what is published is what was built. +A nightly promotion changes no file in this repository: there is no ref to pin +and no entry to bless, because what is published is what was built. A release +does, and that is the difference between them — the ref it ships is written +down and reviewed before the run that ships it. -#### Moving to `qli` +#### Moving the nightly to `qli` -`qli-staging` is the current destination, set by `DEBUSINE_STAGING_WORKSPACE` +`qli-staging` is the nightly's destination, set by `DEBUSINE_STAGING_WORKSPACE` and defaulted in [daily.yml](.github/workflows/daily.yml). Pointing that variable at `qli` would publish every night's kernel straight into the released -archive with nothing in between, so it is not a change to make on its own. -Moving the `promote` job to the **Production** environment first is what -restores an approval gate — at the cost of every nightly run stopping to wait -for one. A pipeline that wants both unattended nightlies and a gated `qli` -needs the two archives it has today, and a promotion step between them that can -run later; that step in turn needs a durable source, which is what -`qli-staging` is for. +archive with nothing in between, and the approval gate that +`promote-environment` provides would then have to be applied to every nightly +run — stopping each of them to wait for one. The two archives, with +`release.yml` between them, are what keep nightlies unattended and `qli` gated. ## Matrix Model -`ci/build-matrix.yaml` is a mapping with exactly one top-level key, `builds`. -One entry in it is one generated package, so there is no +`ci/build-matrix.yaml` is a mapping with two top-level keys, `builds` and +`releases`. One entry in either is one generated package, so there is no expansion step: `ci/scripts/resolve-matrix.py` validates the whole document, selects the entries matching the requested names, variant, and suite, and hands -them to the workflow matrix as they stand. Each entry carries: +them to the workflow matrix as they stand. + +The two lists have the same shape and are validated separately, so a name may +appear in both and means the same build in each. `builds` is what Daily runs +nightly; `releases` is what `release.yml` can ship, selected with +`--releases`. Both are validated on every invocation whichever is asked for, so +a broken release entry fails a nightly run rather than waiting to be found by +whoever next tries to release. Each entry carries: | Field | Purpose | | --- | --- | @@ -226,6 +265,7 @@ them to the workflow matrix as they stand. Each entry carries: | `debian_revision` | The Debian revision this package is built at, stated outright. Carried into the archive as built, because publishing promotes the artifact rather than rebuilding it. | | `localversion`, `kver_extra` | Optional version overrides forwarded to packaging. | | `debusine_parent_workspace` | Optional parent workspace override for the variant's CI child workspaces. | +| `target_workspace` | **`releases` only, and required there.** The Debusine workspace this entry publishes into. It is the one field a `builds` entry may not carry: where a nightly goes follows from why it is running, and is the calling workflow's to decide, while a release exists precisely to put one ref into one archive. | `resolve-matrix.py` rejects the matrix — before any build job starts — where an entry has an unknown field or a missing required one, a malformed variant or @@ -234,6 +274,10 @@ suite identifier, an unknown `ref_strategy`, a `tag_pattern` without or collides with another fragment's filename, a `dkms` entry that is not a package name stem or repeats, or a `debian_revision` that is not a valid Debian revision. +A `releases` entry is held to two rules of its own: `ref_strategy` must be +`pinned_ref`, so releasing twice releases the same kernel, and `suite` must be +a Debian one, because only that path can promote into a workspace. + Because entries are written out in full, the resolver also checks the invariants that span them, which is what makes the duplication safe to read at face value: @@ -306,7 +350,8 @@ flowchart LR DEB --> S3["Download .deb files\nPublish to S3"] DEB --> TW{"Which caller"} - TW -->|"daily.yml"| STG["Promote to qli-staging"] + TW -->|"daily.yml"| STG["Promote to qli-staging\nStaging environment"] + TW -->|"release.yml"| REL["Promote to qli\nProduction environment"] TW -->|"pr-build.yml"| NONE["No archive"] UBU --> US3["Publish .deb files to S3"] @@ -321,6 +366,7 @@ flowchart TD subgraph triggers[Triggers] A1["daily.yml\nScheduled full matrix"] A2["daily.yml\nManual: all or named builds"] + A3["release.yml\nManual: pinned refs from releases"] A5["pr-build.yml\nFull matrix on every PR"] end @@ -341,10 +387,12 @@ flowchart TD subgraph outputs[Outputs] D1["S3 artifacts"] D3["qli-staging APT repository"] + D4["qli APT repository"] end A1 --> B1 A2 --> B1 + A3 --> B1 A5 --> B1 B1 --> B2 & B5 B2 --> C2 @@ -353,11 +401,14 @@ flowchart TD C3 --> C5 & C6 C4 --> D1 C5 --> D1 - C6 --> D3 + C6 --> D3 & D4 ``` A leg runs every job drawn under it except `promote`, which only a caller -naming a workspace reaches — so a PR build stops at S3. +naming a workspace reaches — so a PR build stops at S3. Which archive `promote` +reaches, and whether it waits for an approval first, is that caller's too: the +nightly goes to `qli-staging` unattended, and a release to `qli` through the +Production environment. ### Prepare stage @@ -396,7 +447,7 @@ flowchart LR subgraph promote[promote job: only when the caller named a workspace] WS --> PROMOTE["lib/release\nStart package-publish"] - PROMOTE --> STG["qli-staging\nDebusine APT repository"] + PROMOTE --> STG["qli-staging (daily)\nqli (release)\nDebusine APT repository"] end ``` @@ -490,6 +541,20 @@ workflows themselves (`build-kernel-debian.yml`, `build-kernel-ubuntu.yml`) are `workflow_call` only and cannot be dispatched: one run of each is one matrix entry, and a reusable workflow cannot fan itself out over a list. +### Releasing + +Use **Actions** → **release** → **Run workflow**. It takes the same **Builds** +field, selecting from the matrix's `releases` list rather than from `builds`, +and nothing else — the ref, the packaging and the destination archive are all +the entry's. + +Release a kernel by opening a pull request that sets `branch_or_tag` on the +entries being released, merging it, and dispatching `release` for those names. +The run stops at the `promote` job for the Production environment's approval, +having already built, published to S3 and printed what it is about to release +in the run summary, so the approval is given against the refs and versions in +front of you. + ## Configuration ### Repository and organization variables @@ -500,7 +565,7 @@ entry, and a reusable workflow cannot fan itself out over a list. | `DEBUSINE_HOST` | Production Debusine host. | | `DEBUSINE_SCOPE` | Debusine scope. | | `DEBUSINE_PARENT_WORKSPACE` | Parent workspace used to create per-run CI child workspaces. | -| `DEBUSINE_STAGING_WORKSPACE` | Workspace the nightly build promotes into. Defaults to `qli-staging`; see [Moving to `qli`](#moving-to-qli) before changing it. | +| `DEBUSINE_STAGING_WORKSPACE` | Workspace the nightly build promotes into. Defaults to `qli-staging`; see [Moving the nightly to `qli`](#moving-the-nightly-to-qli) before changing it. A release names its workspace in the matrix instead, so this does not affect it. | ### Secrets @@ -509,15 +574,17 @@ entry, and a reusable workflow cannot fan itself out over a list. | `DEBUSINE_USER` | Repository | User for Debusine archive and signing-key access. | | `DEBUSINE_TOKEN` | Repository | Token for Debusine build and artifact operations, including the nightly promotion into `qli-staging`. | -The `build` and `promote` jobs of `build-kernel-debian.yml` select the -**Staging** GitHub environment, which is what lets the nightly build promote -unattended. `DEBUSINE_TOKEN` therefore needs write access to the workspace -`DEBUSINE_STAGING_WORKSPACE` names. +The `build` job of `build-kernel-debian.yml` always selects the **Staging** +GitHub environment. The `promote` job selects whichever its caller names +through `promote-environment`: **Staging** for the nightly, which is what lets +it promote unattended, and **Production** for a release, which is what makes it +wait for an approval. -No workflow reads a production release credential today, because nothing -publishes to `qli`. `DEBUSINE_RELEASE_TOKEN` and the **Production** environment -are left configured for when something does; see -[Moving to `qli`](#moving-to-qli). +`DEBUSINE_TOKEN` is what both promotions authenticate with, so it needs write +access to `DEBUSINE_STAGING_WORKSPACE` and to every `target_workspace` the +`releases` list names — `qli` included. `DEBUSINE_RELEASE_TOKEN` is not read by +any workflow; a release is gated by the Production environment rather than by a +credential of its own. ## Maintaining the Matrix @@ -551,6 +618,12 @@ To add a new suite (for an existing or new variant): others (see the ordering discussion in [Overview](#overview)), and so it does not collide with another entry building the same `srcpkg`. +To make a variant releasable, add the matching entries to `releases`: the same +fields, plus `target_workspace`, with `ref_strategy: pinned_ref` and +`branch_or_tag` naming the ref that ships. A variant with no `releases` entries +is built nightly and never shipped, which is the right state for a topic branch +or a tracking build. + No workflow dispatch choices need to be updated: the daily and release dispatches take build names as free text, so a new entry is dispatchable by name, and is picked up by `all`, as soon as it is merged. diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index 36ebc9d8..be95bb4c 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -29,11 +29,24 @@ # target archive carries: the Debian suites bundle camx and iris-vpu, which # Ubuntu does not package. # -# Every entry is built nightly. Where a build is published is not stated here: -# the workflow that runs the build decides it, because it depends on why the -# build is running rather than on what is being built. The nightly promotes its -# Debian entries into the archive; a PR build of the same entry promotes -# nowhere. So an entry describes a kernel, and never a destination. +# Every entry under "builds" is built nightly. Where such a build is published +# is not stated on it: the workflow that runs the build decides it, because it +# depends on why the build is running rather than on what is being built. The +# nightly promotes its Debian entries into the archive; a PR build of the same +# entry promotes nowhere. So a build entry describes a kernel, and never a +# destination. +# +# "releases" is the exception, and the one place a destination is written down. +# A release is a decision rather than a schedule: nothing runs these entries +# until someone dispatches release.yml, and what they name is an immutable ref +# rather than whatever the branch has reached. Each states the workspace it +# publishes into, because that is the whole of what distinguishes it from the +# nightly build of the same flavour -- so the ref that ships and the archive it +# ships into are both reviewable in a pull request, rather than typed into a +# dispatch form at the moment of releasing. +# +# The two lists are validated separately and never mix: a name may appear in +# both, and means the same build in each. # # debian_revision is stated outright, not derived. It carries no marker for # where a build is in its life: the version is decided once, when the package is @@ -236,3 +249,96 @@ builds: # kernel_config: [] # dkms: [] # debian_revision: '0qli1' + +# Releases. Dispatched from release.yml and never scheduled, so nothing here +# runs until someone asks for it by name. +# +# Every entry pins its ref: ref_strategy must be pinned_ref, because a release +# names the exact source that ships. latest_tag would make two dispatches of +# one entry release different kernels, and branch_tip would make them release +# whatever the branch had reached. +# +# Debian suites only. Promotion runs through Debusine, and the Ubuntu path +# publishes to S3 without a workspace to promote into, so an Ubuntu entry here +# would name a destination nothing could deliver it to. +# +# Updating a release means editing branch_or_tag in a pull request, which is +# what puts the ref that ships under review. +releases: + # --------------------------------------------------------------------------- + # qcom-next 7.2 — the standard kernel. + # --------------------------------------------------------------------------- + - name: qcom-next-trixie + suite: trixie + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-20260826 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1' + target_workspace: qli + + - name: qcom-next-forky + suite: forky + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-20260826 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1' + target_workspace: qli + + # --------------------------------------------------------------------------- + # qcom-next-debug 7.2 — the same ref with the in-tree debug fragments. + # --------------------------------------------------------------------------- + - name: qcom-next-debug-trixie + suite: trixie + flavour: qcom-next-debug + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-20260826 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next-debug + binpkg: linux-image-qcom-next-debug + kernel_config: + - intree:arch/arm64/configs/qcom_debug.config + - intree:kernel/configs/debug.config + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1~bpo13+1' + target_workspace: qli + + - name: qcom-next-debug-forky + suite: forky + flavour: qcom-next-debug + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next-7.2-20260826 + ref_strategy: pinned_ref + srcpkg: linux-qcom-next-debug + binpkg: linux-image-qcom-next-debug + kernel_config: + - intree:arch/arm64/configs/qcom_debug.config + - intree:kernel/configs/debug.config + dkms: + - kgsl + - camx + - iris-vpu + - audioreach + debian_revision: '0qli1' + target_workspace: qli diff --git a/ci/scripts/resolve-matrix.py b/ci/scripts/resolve-matrix.py index 0a116b21..b3532c30 100755 --- a/ci/scripts/resolve-matrix.py +++ b/ci/scripts/resolve-matrix.py @@ -9,14 +9,18 @@ script validates the whole document, selects the entries a caller asked for, and prints them. -There is one kind of entry, and it describes a kernel rather than a -destination. Where a build is published is decided by the workflow running it, -not stated here: the nightly promotes its Debian entries into the archive, and -a PR build promotes nowhere. - -The document is validated in full on every invocation, not just the selected +The document has two lists of entries, both of the same shape. "builds" is what +is built nightly, and describes a kernel rather than a destination: where such +a build is published is decided by the workflow running it, not stated here -- +the nightly promotes its Debian entries into the archive, and a PR build +promotes nowhere. "releases" is the exception, selected with --releases, and +each of its entries names the workspace it publishes into and pins the exact +ref that ships. + +Both lists are validated in full on every invocation, not just the selected entries, so a typo in an entry nobody selected fails the run that would have -built its siblings rather than lying in wait. +built its siblings rather than lying in wait. They are validated separately and +never mix: a name may appear in both, and means the same build in each. Usage: ci/scripts/resolve-matrix.py @@ -25,8 +29,14 @@ ci/scripts/resolve-matrix.py --flavour qcom-next ci/scripts/resolve-matrix.py --family ubuntu --allow-empty ci/scripts/resolve-matrix.py --build qcom-next-trixie --field debian_revision + ci/scripts/resolve-matrix.py --releases + ci/scripts/resolve-matrix.py --releases --build qcom-next-trixie Options: + --releases Select from the releases list rather than from + builds. Release entries carry a + target_workspace and pin their ref, so this is + how release.yml asks what may be released. --build NAMES Select only these builds by name, comma-separated. A name identifies one build, so this is the way to ask for a specific set of them. @@ -122,10 +132,18 @@ def family_for(suite): "debusine_parent_workspace", ) +# A release entry states its destination, and a build entry may not. This is +# the whole difference between the two lists: where a nightly build is +# published follows from why it is running, so its entry says nothing about it, +# while a release exists precisely to put one ref into one archive. +RELEASE_REQUIRED_STRING_FIELDS = ("target_workspace",) + KNOWN_FIELDS = frozenset( REQUIRED_STRING_FIELDS + OPTIONAL_STRING_FIELDS + ("kernel_config", "dkms") ) +RELEASE_KNOWN_FIELDS = KNOWN_FIELDS | frozenset(RELEASE_REQUIRED_STRING_FIELDS) + # Everything about what is built is anchored on flavour, not on the build's # name. The flavour is the kernel's own identity: it becomes the LOCALVERSION # suffix, so two flavours built from one ref produce distinct kernel releases @@ -150,9 +168,10 @@ def family_for(suite): NAME_RE = re.compile(r"^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$") -# The dispatch form of daily.yml takes one builds field, where "all" means -# every entry and anything else is a list of names. A build actually called all -# would be unreachable through it, so the matrix may not define one. +# The dispatch forms of daily.yml and release.yml take one builds field, where +# "all" means every entry and anything else is a list of names. A build +# actually called all would be unreachable through them, so the matrix may not +# define one. RESERVED_NAMES = ("all",) # A Debian revision: no hyphen (that would start a new revision component) and @@ -247,9 +266,15 @@ def check_dkms(entry, report): report(f"dkms entry '{module}' must be a package name stem, e.g. kgsl") -def check_entry(entry, report): +def check_entry(entry, report, release=False): """Validate one delivery entry in isolation.""" - for field in REQUIRED_STRING_FIELDS: + required = REQUIRED_STRING_FIELDS + known = KNOWN_FIELDS + if release: + required += RELEASE_REQUIRED_STRING_FIELDS + known = RELEASE_KNOWN_FIELDS + + for field in required: value = entry.get(field) if not isinstance(value, str) or not value: report(f"missing or invalid {field}") @@ -258,7 +283,10 @@ def check_entry(entry, report): if field in entry and not isinstance(entry[field], str): report(f"invalid {field}") - for field in sorted(set(entry) - KNOWN_FIELDS): + for field in sorted(set(entry) - known): + # target_workspace on a build entry is the likely version of this, and + # says something the nightly would silently ignore: a build entry can + # no more choose an archive than it can choose to run. report(f"unknown field {field}") for field in ("name", "suite", "flavour"): @@ -292,6 +320,28 @@ def check_entry(entry, report): "(letters, digits, and . + ~ only, starting with a letter or digit)" ) + if not release: + return + + # A release names the exact source that ships. latest_tag would make two + # dispatches of one entry release different kernels, and branch_tip would + # make them release whatever the branch had reached by then. + if ref_strategy in REF_STRATEGIES and ref_strategy != "pinned_ref": + report( + f"ref_strategy must be pinned_ref for a release (got {ref_strategy}); " + "a release names one immutable ref" + ) + + # Promotion runs through Debusine, which is the Debian path. The Ubuntu + # path publishes to S3 and has no workspace to promote into, so an entry + # here would name a destination nothing could deliver it to. + suite = entry.get("suite") + if isinstance(suite, str) and suite and family_for(suite) != "debian": + report( + f"suite {suite} is not a Debian suite; only the Debian path can " + "promote into a workspace, so only its builds can be released" + ) + def describe(entry, index): """Label an entry in an error message by what identifies it to a reader.""" @@ -403,21 +453,33 @@ def check_consistency(builds, errors): ) -def validate(builds): - """Return every problem found in the matrix, as a list of messages.""" +def validate(builds, release=False): + """Return every problem found in one list of entries, as messages.""" errors = [] for index, entry in enumerate(builds): if not isinstance(entry, dict): errors.append(f"{describe(entry, index)}: delivery entries must be mappings") continue label = describe(entry, index) - check_entry(entry, lambda message, label=label: errors.append(f"{label}: {message}")) + check_entry( + entry, + lambda message, label=label: errors.append(f"{label}: {message}"), + release=release, + ) check_consistency(builds, errors) return errors -def load_matrix(path): - """Read, parse, and validate the matrix, returning its builds.""" +ROOT_KEYS = ("builds", "releases") + + +def load_matrix(path, section="builds"): + """Read and validate the whole matrix, returning one section's entries. + + Both sections are validated however few of them the caller wants, so a + broken release entry fails a nightly run too -- rather than waiting to be + found by whoever next tries to release. + """ try: with open(path, encoding="utf-8") as handle: document = yaml.safe_load(handle) @@ -431,36 +493,56 @@ def load_matrix(path): if not isinstance(document, dict): sys.exit(f"ERROR: {path}: matrix root must be a mapping with a builds key") - # builds is the whole schema. Anything else at the root is a leftover from - # an older matrix (suite_suffix_mapping, say) that would otherwise sit - # there looking authoritative while nothing read it. - unknown_root = sorted(set(document) - {"builds"}) + # builds and releases are the whole schema. Anything else at the root is a + # leftover from an older matrix (suite_suffix_mapping, say) that would + # otherwise sit there looking authoritative while nothing read it. + unknown_root = sorted(set(document) - set(ROOT_KEYS)) if unknown_root: sys.exit( f"ERROR: {path}: unknown top-level key(s) {', '.join(unknown_root)}; " - "builds is the only one" + + " and ".join(ROOT_KEYS) + " are the only ones" ) - builds = document.get("builds") - if not isinstance(builds, list): - sys.exit(f"ERROR: {path}: builds must be a list") - if not builds: - sys.exit(f"ERROR: {path}: builds must contain at least one entry") + sections = {} + for key in ROOT_KEYS: + entries = document.get(key) + # releases is optional, and a matrix with none is a matrix nothing has + # blessed yet rather than a broken one. builds is not: a matrix that + # builds nothing is a mistake in every case. + if entries is None and key != "builds": + sections[key] = [] + continue + if not isinstance(entries, list): + sys.exit(f"ERROR: {path}: {key} must be a list") + if not entries and key == "builds": + sys.exit(f"ERROR: {path}: builds must contain at least one entry") + sections[key] = entries - errors = validate(builds) + errors = [] + for key, entries in sections.items(): + errors += [ + f"{key}: {error}" for error in validate(entries, release=key == "releases") + ] if errors: sys.exit( f"ERROR: Invalid kernel delivery matrix in {path}:\n" + "\n".join(f" - {error}" for error in errors) ) + selected = sections[section] + if not selected: + sys.exit( + f"ERROR: {path}: no {section} are defined, so there is nothing to " + "select from" + ) + # Derived, never written: family follows from suite, so the matrix cannot # state one that disagrees with the suite it is built for. Attached after # validation, which rejects family as an unknown field on an entry. - for entry in builds: + for entry in selected: entry["family"] = family_for(entry["suite"]) - return builds + return selected def parse_filter(value): @@ -532,6 +614,11 @@ def main(): description="Validate and select entries from the kernel delivery matrix.", epilog="See the module docstring in this file for full documentation.", ) + parser.add_argument( + "--releases", + action="store_true", + help="select from the releases list rather than from builds", + ) parser.add_argument( "--build", default="", @@ -564,7 +651,9 @@ def main(): ) args = parser.parse_args() - builds = load_matrix(args.matrix_file) + builds = load_matrix( + args.matrix_file, section="releases" if args.releases else "builds" + ) filters = { "name": parse_filter(args.build), "flavour": parse_filter(args.flavour), From b2f8f5709a2e39be68d991ce3c9feb0cb6a2c5f9 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 49/71] wip Signed-off-by: Christopher Obbard --- ci/build-matrix.yaml | 142 +++++++++++++++++++++---------------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index be95bb4c..454a5d95 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -268,77 +268,77 @@ releases: # --------------------------------------------------------------------------- # qcom-next 7.2 — the standard kernel. # --------------------------------------------------------------------------- - - name: qcom-next-trixie - suite: trixie - flavour: qcom-next - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next-7.2-20260826 - ref_strategy: pinned_ref - srcpkg: linux-qcom-next - binpkg: linux-image-qcom-next - kernel_config: [] - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1~bpo13+1' - target_workspace: qli + # - name: qcom-next-trixie + # suite: trixie + # flavour: qcom-next + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next-7.2-20260826 + # ref_strategy: pinned_ref + # srcpkg: linux-qcom-next + # binpkg: linux-image-qcom-next + # kernel_config: [] + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1~bpo13+1' + # target_workspace: qli - - name: qcom-next-forky - suite: forky - flavour: qcom-next - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next-7.2-20260826 - ref_strategy: pinned_ref - srcpkg: linux-qcom-next - binpkg: linux-image-qcom-next - kernel_config: [] - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1' - target_workspace: qli + # - name: qcom-next-forky + # suite: forky + # flavour: qcom-next + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next-7.2-20260826 + # ref_strategy: pinned_ref + # srcpkg: linux-qcom-next + # binpkg: linux-image-qcom-next + # kernel_config: [] + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1' + # target_workspace: qli - # --------------------------------------------------------------------------- - # qcom-next-debug 7.2 — the same ref with the in-tree debug fragments. - # --------------------------------------------------------------------------- - - name: qcom-next-debug-trixie - suite: trixie - flavour: qcom-next-debug - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next-7.2-20260826 - ref_strategy: pinned_ref - srcpkg: linux-qcom-next-debug - binpkg: linux-image-qcom-next-debug - kernel_config: - - intree:arch/arm64/configs/qcom_debug.config - - intree:kernel/configs/debug.config - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1~bpo13+1' - target_workspace: qli + # # --------------------------------------------------------------------------- + # # qcom-next-debug 7.2 — the same ref with the in-tree debug fragments. + # # --------------------------------------------------------------------------- + # - name: qcom-next-debug-trixie + # suite: trixie + # flavour: qcom-next-debug + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next-7.2-20260826 + # ref_strategy: pinned_ref + # srcpkg: linux-qcom-next-debug + # binpkg: linux-image-qcom-next-debug + # kernel_config: + # - intree:arch/arm64/configs/qcom_debug.config + # - intree:kernel/configs/debug.config + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1~bpo13+1' + # target_workspace: qli - - name: qcom-next-debug-forky - suite: forky - flavour: qcom-next-debug - git_clone: https://github.com/qualcomm-linux/kernel - branch_or_tag: qcom-next-7.2-20260826 - ref_strategy: pinned_ref - srcpkg: linux-qcom-next-debug - binpkg: linux-image-qcom-next-debug - kernel_config: - - intree:arch/arm64/configs/qcom_debug.config - - intree:kernel/configs/debug.config - dkms: - - kgsl - - camx - - iris-vpu - - audioreach - debian_revision: '0qli1' - target_workspace: qli + # - name: qcom-next-debug-forky + # suite: forky + # flavour: qcom-next-debug + # git_clone: https://github.com/qualcomm-linux/kernel + # branch_or_tag: qcom-next-7.2-20260826 + # ref_strategy: pinned_ref + # srcpkg: linux-qcom-next-debug + # binpkg: linux-image-qcom-next-debug + # kernel_config: + # - intree:arch/arm64/configs/qcom_debug.config + # - intree:kernel/configs/debug.config + # dkms: + # - kgsl + # - camx + # - iris-vpu + # - audioreach + # debian_revision: '0qli1' + # target_workspace: qli From 5671553424be4a0b027a1bf14434796f8fe1b8b0 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 50/71] ci: say why the family guards exist and why a skipped check reads oddly The if: guards on the build jobs look removable, and are not: an empty matrix is a workflow error rather than an empty job list, so a family the run selects nothing on has to be skipped rather than started with nothing to do. Skipping one has a visible cost that looks like a bug. GitHub never expands a job it skips, so it cannot evaluate name: ${{ matrix.name }} and shows the expression's source text -- a check literally called "matrix.name". A || fallback does not help, printing its own source text just the same, and the only name that would read well is one never mentioning matrix, which would mean renaming every check in every workflow to tidy up one that appears when a family is empty. Both facts are written down so the next reader reaches that conclusion without rediscovering it. Signed-off-by: Christopher Obbard --- .github/workflows/daily.yml | 12 +++++++++--- .github/workflows/pr-build.yml | 7 ++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 8c0989ac..6ee6bab4 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -109,9 +109,15 @@ jobs: # called. fail-fast is disabled so a failed trixie build does not cancel the # forky build. # - # The if: guards only matter for a narrowed dispatch that selects nothing on - # one path; an empty matrix is an error rather than an empty job list. The - # scheduled full-matrix run has entries on both. + # The if: guards catch a family the run selects nothing on -- a narrowed + # dispatch, or a matrix with no entries for it. They are not optional: an + # empty matrix is a workflow error rather than an empty job list. + # + # A skipped family shows up as a check literally named "matrix.name": GitHub + # never expands a job it skips, so it prints the expression's source text, + # and a || fallback prints just the same. Only a name that never mentions + # matrix would read well, which would mean renaming every check in every + # workflow to tidy up one that appears when a family is empty. So it stands. # --------------------------------------------------------------------------- build-debian: name: ${{ matrix.name }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 186e747a..8f08cee2 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -58,9 +58,10 @@ jobs: # build-debian / build-ubuntu: one parallel job per matrix entry, mirroring # daily.yml. # - # A PR gets one check per build leg and nothing else: neither workflow has a - # branch to skip, and a PR builds the whole matrix, so both families have - # entries and neither guard fires. + # A PR builds the whole matrix, so it gets one check per build leg and + # nothing else -- unless the matrix has no entries for one family, which + # fires that family's guard and leaves a check named "matrix.name". + # daily.yml documents why the guard is needed and the name left alone. # # fail-fast is disabled so a broken trixie leg still leaves forky and # resolute results on the PR. From fecd18287617768c5fecde5732c991e6525a9086 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 51/71] scripts: clone the kernel over HTTPS by default build-kernel.sh defaulted to the SSH clone URL while CI and the matrix use HTTPS, so a local build needed a GitHub key that a read-only clone does not. The same URL is what the changelog records as the source, so the default now matches what ci/build-matrix.yaml says. Signed-off-by: Christopher Obbard --- build-kernel.sh | 2 +- debian/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-kernel.sh b/build-kernel.sh index 23375748..909af185 100755 --- a/build-kernel.sh +++ b/build-kernel.sh @@ -8,7 +8,7 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -DEFAULT_REPO="git@github.com:qualcomm-linux/kernel.git" +DEFAULT_REPO="https://github.com/qualcomm-linux/kernel" DEFAULT_BRANCH="qcom-next" DEFAULT_DISTRO="trixie" DEFAULT_BUILD_MODE="docker" diff --git a/debian/README.md b/debian/README.md index a3a77af7..9a5380c8 100644 --- a/debian/README.md +++ b/debian/README.md @@ -242,7 +242,7 @@ clone → prepare → build. Run it from the repo root. ```bash # 1. Set up kernel source -git clone git@github.com:qualcomm-linux/kernel.git kernel-source +git clone https://github.com/qualcomm-linux/kernel kernel-source cd kernel-source git checkout qcom-next From a8608509c5c2acddb3de8773dabff74c39cdc198 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 52/71] ci: drop the PR-patch inputs nothing passes prepare-kernel-source took qcom-next-pr and kernel-topics-pr and merged or applied them after the clone, but no workflow passes either: the build workflows have no input to carry them, so the two steps could never run. They could not have been wired up as they stood, either. The commit SHA and date are read from HEAD before the patches would be applied, so a patched build would name a commit it did not build. And a merge commit made at build time has no stable SHA, so the version could never be reproduced from it. An orig tarball keyed on the commit, which the source package is about to be, rules such builds out for good. Signed-off-by: Christopher Obbard --- .../actions/prepare-kernel-source/action.yml | 59 +------------------ 1 file changed, 2 insertions(+), 57 deletions(-) diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index 5b2f45df..299f0115 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -1,7 +1,7 @@ name: Prepare kernel source description: > - Resolve the kernel ref, clone it, apply any PR patches, inject debian/ and - generate the source package, then upload the prepared tree as an artifact. + Resolve the kernel ref, clone it, inject debian/ and generate the source + package, then upload the prepared tree as an artifact. # Every build takes this path, whichever family it belongs to, so it lives in a # composite action rather than a reusable workflow: the two family workflows @@ -85,12 +85,6 @@ inputs: kver-extra: description: 'Extra package-version suffix (e.g. -ci42)' default: '' - qcom-next-pr: - description: 'qcom-next PR numbers to merge, space-separated' - default: '' - kernel-topics-pr: - description: 'kernel-topics PR numbers to apply as patches, space-separated' - default: '' runs: using: composite @@ -327,55 +321,6 @@ runs: fi echo "debian_revision=$DEBIAN_REVISION" >> "$GITHUB_OUTPUT" - - name: Merge qcom-next PR patches - if: ${{ inputs.qcom-next-pr != '' }} - shell: bash - env: - QCOM_NEXT_PR: ${{ inputs.qcom-next-pr }} - KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} - run: | - cd "$KERNEL_DIR" - echo "Merging qcom-next PR(s): $QCOM_NEXT_PR" - for pr in $QCOM_NEXT_PR; do - [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid qcom-next PR number: $pr"; exit 1; } - echo "::group::Merging qcom-next PR #$pr" - git fetch --no-tags origin "pull/$pr/head:pr-$pr" - if ! git merge --no-ff --no-commit "pr-$pr"; then - echo "ERROR: Merge conflict while merging PR #$pr. Aborting." - git merge --abort || true - exit 1 - fi - if ! git diff --cached --quiet; then - git commit -m "Merged qcom-next PR #$pr" - echo "PR #$pr merged successfully." - else - echo "PR #$pr already present; nothing to merge." - fi - echo "::endgroup::" - done - - - name: Apply kernel-topics PR patches - if: ${{ inputs.kernel-topics-pr != '' }} - shell: bash - env: - KERNEL_TOPICS_PR: ${{ inputs.kernel-topics-pr }} - KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} - run: | - cd "$KERNEL_DIR" - echo "Applying kernel-topics PR patch(es): $KERNEL_TOPICS_PR" - for pr in $KERNEL_TOPICS_PR; do - [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid kernel-topics PR number: $pr"; exit 1; } - echo "::group::Applying kernel-topics PR #$pr" - wget -q "https://github.com/qualcomm-linux/kernel-topics/pull/$pr.patch" -O "$pr.patch" - if ! git am "$pr.patch"; then - echo "ERROR: Patch application failed for PR #$pr. Aborting." - git am --abort || true - exit 1 - fi - echo "PR #$pr applied successfully." - echo "::endgroup::" - done - - name: Prepare source shell: bash env: From e5889cc33957ac67bc2769e6417dc85baac513d5 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 53/71] ci: require debian-revision in prepare and drop the matrix lookup The action looked the revision up in ci/build-matrix.yaml when its input was empty, for a caller with no matrix entry in hand. No such caller exists: the build workflows are workflow_call only, are called with a matrix entry, and give the input a default of their own, so the lookup could not be reached. The revision is now a required input, and a caller that forgets it fails at the action boundary rather than building at a revision nobody chose. resolve-matrix.py keeps --field, which is still a useful way to ask the matrix a question from a shell. Signed-off-by: Christopher Obbard --- .../actions/prepare-kernel-source/action.yml | 36 ++----------------- README.md | 8 ++--- 2 files changed, 6 insertions(+), 38 deletions(-) diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index 299f0115..9e2903ae 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -77,8 +77,8 @@ inputs: description: 'Out-of-tree DKMS modules to bundle, comma-separated and without the -dkms suffix' default: '' debian-revision: - description: 'Debian revision override (empty takes the revision the matrix gives this build)' - default: '' + description: 'Debian revision component of the package version, from the matrix entry' + required: true localversion: description: 'LOCALVERSION override (auto-derived from the resolved ref if empty)' default: '' @@ -291,36 +291,6 @@ runs: done >> "$GITHUB_OUTPUT" fi - - name: Resolve Debian revision - # The build workflows pass the matrix entry's own debian-revision - # straight through, so this lookup is the fallback for a caller that has - # no entry in hand and leaves the input empty: it looks up the entry for - # the build name and suite it was given, and builds at the revision the - # nightly build uses. - id: revision - shell: bash - env: - DEBIAN_REVISION_INPUT: ${{ inputs.debian-revision }} - BUILD_INPUT: ${{ inputs.build }} - DISTRO: ${{ steps.env.outputs.distro }} - run: | - set -euo pipefail - if [[ -n "$DEBIAN_REVISION_INPUT" ]]; then - DEBIAN_REVISION="$DEBIAN_REVISION_INPUT" - echo "Using supplied Debian revision: $DEBIAN_REVISION" - else - DEBIAN_REVISION=$(ci/scripts/resolve-matrix.py \ - --build "$BUILD_INPUT" \ - --suite "$DISTRO" \ - --field debian_revision) || { - echo "::error::No matrix entry named $BUILD_INPUT for suite $DISTRO." - echo "::error::Add one to ci/build-matrix.yaml, or set the debian-revision input to build this combination ad hoc." - exit 1 - } - echo "Matrix Debian revision for $BUILD_INPUT/$DISTRO: $DEBIAN_REVISION" - fi - echo "debian_revision=$DEBIAN_REVISION" >> "$GITHUB_OUTPUT" - - name: Prepare source shell: bash env: @@ -334,7 +304,7 @@ runs: DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} LOCALVERSION: ${{ steps.localversion.outputs.localversion }} SNAPSHOT: ${{ steps.localversion.outputs.snapshot }} - DEBIAN_REVISION: ${{ steps.revision.outputs.debian_revision }} + DEBIAN_REVISION: ${{ inputs.debian-revision }} KERNEL_URL: ${{ inputs.kernel-url }} KERNEL_REF: ${{ steps.ref.outputs.kernel_ref }} KERNEL_SHA: ${{ steps.clone.outputs.kernel_sha }} diff --git a/README.md b/README.md index 610717f6..f3d33cdf 100644 --- a/README.md +++ b/README.md @@ -295,11 +295,9 @@ face value: - No `srcpkg` or `binpkg` is shared between flavours, and no two entries build the same `srcpkg` at the same `debian_revision`. -The build workflows pass their entry's own `debian_revision` through. A caller -with no entry in hand can leave the `debian-revision` input empty, and the -`prepare-kernel-source` action looks up the entry for the build name and suite -it was given (`resolve-matrix.py --field debian_revision`) and builds at the -revision the nightly build uses. +The build workflows pass their entry's own `debian_revision` through, and the +`prepare-kernel-source` action requires it: there is no caller without an +entry in hand, so there is no fallback to look one up. Each entry has a distinct prepared-source artifact, Debusine child workspace, and S3 path keyed by `flavour + suite`. This prevents two flavours that both From 355924818a2d502ccc3dfba33638a732a703568c Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 54/71] ci: require the inputs that describe the build Every input naming what a build is -- the entry, its suite and flavour, the kernel repository and ref, the package names and the Debian revision -- had a default spelling out the qcom-next entry. No caller relied on one: daily.yml, pr-build.yml and release.yml pass all of them from the matrix. What the defaults did was let a caller that dropped one build the default entry's kernel under another entry's name without anything noticing. They are now required, so an omission is a workflow error at parse time. The inputs that keep a default are the ones for which empty is a value: tag-pattern outside latest_tag, an empty fragment or module list, and the overrides and destinations the caller chooses rather than the entry. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 18 +++++++++--------- .github/workflows/build-kernel-ubuntu.yml | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 739224f6..94bbe49b 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -35,27 +35,27 @@ on: build: description: 'Name of this build, as listed in ci/build-matrix.yaml' type: string - default: 'qcom-next-trixie' + required: true flavour: description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' type: string - default: 'qcom-next' + required: true suite: description: 'Target Debian suite (trixie, forky, sid)' type: string - default: 'trixie' + required: true kernel-branch: description: 'Source branch or pinned tag used by branch_tip or pinned_ref' type: string - default: 'qcom-next' + required: true git-clone: description: 'Kernel repository URL' type: string - default: 'https://github.com/qualcomm-linux/kernel' + required: true ref-strategy: description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' type: string - default: 'latest_tag' + required: true tag-pattern: description: 'Tag glob used when ref-strategy=latest_tag' type: string @@ -63,11 +63,11 @@ on: srcpkg: description: 'Source package name' type: string - default: 'linux-qcom-next' + required: true binpkg: description: 'Binary metapackage name' type: string - default: 'linux-image-qcom-next' + required: true kernel-config: description: 'Extra config fragments applied on top of all of debian/config-available/' type: string @@ -79,7 +79,7 @@ on: debian-revision: description: 'Debian revision component of the package version' type: string - default: '0qli1' + required: true localversion: description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' type: string diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index c0c36423..c8dfe5e1 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -28,27 +28,27 @@ on: build: description: 'Name of this build, as listed in ci/build-matrix.yaml' type: string - default: 'qcom-next-resolute' + required: true flavour: description: 'Kernel flavour: the LOCALVERSION suffix and so the kernel release identity' type: string - default: 'qcom-next' + required: true suite: description: 'Target Ubuntu suite (noble, resolute)' type: string - default: 'resolute' + required: true kernel-branch: description: 'Source branch or pinned tag used by branch_tip or pinned_ref' type: string - default: 'qcom-next' + required: true git-clone: description: 'Kernel repository URL' type: string - default: 'https://github.com/qualcomm-linux/kernel' + required: true ref-strategy: description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' type: string - default: 'latest_tag' + required: true tag-pattern: description: 'Tag glob used when ref-strategy=latest_tag' type: string @@ -56,11 +56,11 @@ on: srcpkg: description: 'Source package name' type: string - default: 'linux-qcom-next' + required: true binpkg: description: 'Binary metapackage name' type: string - default: 'linux-image-qcom-next' + required: true kernel-config: description: 'Extra config fragments applied on top of all of debian/config-available/' type: string @@ -72,7 +72,7 @@ on: debian-revision: description: 'Debian revision component of the package version' type: string - default: '0qli1' + required: true localversion: description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' type: string From f414b89aedc1d745cd57f167ff218374e04b142c Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 55/71] scripts: derive every version field through derive-localversion.sh Three scripts derived LOCALVERSION from a tag, each its own way. CI used ci/scripts/derive-localversion.sh. prepare-source.sh carried a copy of the same rule. build-kernel.sh carried an older one that produced the pre-plus, SHA-less form (qcom-next-20260210), passed no snapshot, and so gave a local build a Debian version with no +git component at all -- a different version from the one CI gives the same commit. prepare-source.sh now calls derive-localversion.sh, with the exact tag or the branch HEAD sits on as the ref and HEAD's committer date for the branch-tip case, which is precisely what the prepare action feeds it. It takes a --flavour for the identity part of LOCALVERSION, defaulting to qcom-next, and build-kernel.sh passes that through and derives nothing of its own. A local build of a commit now produces the version CI would, which is the precondition for a source package built locally being interchangeable with one built in CI. Signed-off-by: Christopher Obbard --- build-kernel.sh | 59 +++++++++++--------------------- prepare-source.sh | 87 +++++++++++++++++++++++++++-------------------- 2 files changed, 69 insertions(+), 77 deletions(-) diff --git a/build-kernel.sh b/build-kernel.sh index 909af185..c3b64fc0 100755 --- a/build-kernel.sh +++ b/build-kernel.sh @@ -11,6 +11,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEFAULT_REPO="https://github.com/qualcomm-linux/kernel" DEFAULT_BRANCH="qcom-next" DEFAULT_DISTRO="trixie" +DEFAULT_FLAVOUR="qcom-next" DEFAULT_BUILD_MODE="docker" KERNEL_DIR="$SCRIPT_DIR/kernel-source" OUTPUT_BASE_DIR="$SCRIPT_DIR/kernel-build" @@ -38,11 +39,14 @@ OPTIONS: -r, --repo URL Kernel repository URL (default: $DEFAULT_REPO) Version control: - --localversion TAG LOCALVERSION suffix (e.g. qcom-next-20260312) - Auto-detected from git tag when using --local-source + --flavour NAME Kernel flavour carried in LOCALVERSION + (default: $DEFAULT_FLAVOUR). Ignored with --localversion. + --localversion SUFFIX LOCALVERSION suffix (e.g. +qcom-next-20260312-g07f50dc44edd) + Derived from the checked-out tag or branch if not given, + by prepare-source.sh, the same way CI derives it. --kver-extra SUFFIX Extra suffix appended to the derived KVER, e.g.: --kver-extra -mybuild - Results in: 7.0.0-rc2-qcom-next-20260312-mybuild + Results in: 7.0.0-rc2+qcom-next-20260312-g07f50dc44edd-mybuild Useful for CI build IDs or local user builds. Build control: @@ -95,7 +99,7 @@ EOF # Defaults TAG=""; LATEST_TAG=false; BRANCH="$DEFAULT_BRANCH"; REPO="$DEFAULT_REPO" -DISTRO="$DEFAULT_DISTRO"; BUILD_MODE="$DEFAULT_BUILD_MODE" +DISTRO="$DEFAULT_DISTRO"; BUILD_MODE="$DEFAULT_BUILD_MODE"; FLAVOUR="$DEFAULT_FLAVOUR" LOCALVERSION=""; KVER_EXTRA=""; PROFILES=""; CLEAN=false LOCAL_SOURCE=""; ENABLE_CONFIGS="squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can"; SKIP_PREPARE=false DKMS_MODULES="" @@ -112,6 +116,7 @@ while [[ $# -gt 0 ]]; do -d|--distro) DISTRO="$2"; shift 2 ;; --local-source) LOCAL_SOURCE="$2"; shift 2 ;; --docker-build) DOCKER_PKG_BUILD="$2"; shift 2 ;; + --flavour) FLAVOUR="$2"; shift 2 ;; --localversion) LOCALVERSION="$2"; shift 2 ;; --kver-extra) KVER_EXTRA="$2"; shift 2 ;; --profiles) PROFILES="$2"; shift 2 ;; @@ -170,7 +175,8 @@ log_step "Configuration:" log_info " Output: $OUTPUT_DIR" log_info " Distro: $DISTRO mode: $BUILD_MODE" [[ "$BUILD_MODE" == "docker" ]] && log_info " Docker build: $DOCKER_PKG_BUILD" -[[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" +[[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" \ + || log_info " Flavour: $FLAVOUR" [[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" [[ -n "$PROFILES" ]] && log_info " Profiles: $PROFILES" [[ -n "$ENABLE_CONFIGS" ]] && log_info " Extra configs: $ENABLE_CONFIGS" @@ -178,17 +184,6 @@ log_info " Distro: $DISTRO mode: $BUILD_MODE" [[ "$SKIP_PREPARE" == true ]] && log_info " Skip prepare: yes (source already prepared by prepare-source.sh)" echo -# ── Helper: derive LOCALVERSION from a tag name ────────────────────────────── -# qcom-next-6.19-rc8-20260210 → qcom-next-20260210 -_auto_localversion() { - local tag="$1" - if [[ "$tag" =~ ^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]+)$ ]]; then - echo "${BASH_REMATCH[1]}-${BASH_REMATCH[2]}" - else - echo "$tag" - fi -} - # ── Git operations: resolve ref → sync → checkout ──────────────────────────── if [[ -z "$LOCAL_SOURCE" ]]; then # Resolve the latest tag remotely before any network I/O (avoids fetching all tags) @@ -229,39 +224,23 @@ if [[ -z "$LOCAL_SOURCE" ]]; then git -C "$KERNEL_DIR" checkout -B "$BRANCH" FETCH_HEAD fi fi - - # Auto-detect LOCALVERSION from tag (applies to both clone and update paths) - if [[ -n "$TAG" && -z "$LOCALVERSION" ]]; then - LOCALVERSION="$(_auto_localversion "$TAG")" - log_info "Auto-detected LOCALVERSION='$LOCALVERSION'" - fi +else + log_info "Using local source as-is (skipping git checkout)" fi cd "$KERNEL_DIR" -# ── Local source: LOCALVERSION detection ───────────────────────────────────── -if [[ -n "$LOCAL_SOURCE" ]]; then - log_info "Using local source as-is (skipping git checkout)" - if [[ -z "$LOCALVERSION" ]]; then - GIT_TAG=$(git describe --tags --exact-match 2>/dev/null || true) - if [[ -n "$GIT_TAG" ]]; then - LOCALVERSION="$(_auto_localversion "$GIT_TAG")" - log_info "Auto-detected LOCALVERSION='$LOCALVERSION' from tag '$GIT_TAG'" - else - log_warn "LOCALVERSION not set and no exact git tag found." - log_warn "Package will be named linux-image--qcom (no branch/ABI suffix)." - log_warn "Use --localversion to specify, e.g.: --localversion qcom-next-20260312" - fi - fi -fi - # ── Source preparation ──────────────────────────────────────────────────────── # Delegates to prepare-source.sh, which is the single source of truth for -# debian/ injection, config fragment activation, and debian/rules prepare. +# debian/ injection, config fragment activation, version derivation, and +# debian/rules prepare. Nothing about the version is decided here: an explicit +# --localversion is passed through, and otherwise prepare-source.sh derives +# it from the checkout the way CI does. # Skipped when --skip-prepare is set (CI mode: prepare-source.sh already ran # as a dedicated prior step). if [[ "$SKIP_PREPARE" != true ]]; then - PREPARE_ARGS=(--source-dir "$KERNEL_DIR" --distro "$DISTRO" --debian-dir "$DEBIAN_DIR") + PREPARE_ARGS=(--source-dir "$KERNEL_DIR" --distro "$DISTRO" --debian-dir "$DEBIAN_DIR" + --flavour "$FLAVOUR") [[ -n "$LOCALVERSION" ]] && PREPARE_ARGS+=(--localversion "$LOCALVERSION") [[ -n "$KVER_EXTRA" ]] && PREPARE_ARGS+=(--kver-extra "$KVER_EXTRA") [[ -n "$ENABLE_CONFIGS" ]] && PREPARE_ARGS+=(--kernel-config "$ENABLE_CONFIGS") diff --git a/prepare-source.sh b/prepare-source.sh index b6219431..b4c69c25 100755 --- a/prepare-source.sh +++ b/prepare-source.sh @@ -20,6 +20,7 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEFAULT_DISTRO="trixie" +DEFAULT_FLAVOUR="qcom-next" DEFAULT_SRCPKG="linux-qcom-next" DEFAULT_BINPKG="linux-image-qcom-next" DEFAULT_DEBIAN_REVISION="0qcom1" @@ -51,14 +52,18 @@ OPTIONS: Version control: -d, --distro DISTRO Target suite: trixie|forky|sid|noble|questing|resolute (default: $DEFAULT_DISTRO) + --flavour NAME Kernel flavour, the identity LOCALVERSION carries + (default: $DEFAULT_FLAVOUR). Only consulted when + --localversion is not given. --localversion SUFFIX LOCALVERSION suffix appended to the base kernel - version (e.g. +qcom-next-20260722). - Auto-detected from git tag if not specified. + version (e.g. +qcom-next-20260722-g07f50dc44edd). + Derived from the checked-out tag or branch by + ci/scripts/derive-localversion.sh if not given. --snapshot SNAPSHOT Dated component of the Debian version: YYYYMMDD with an optional . ordinal (e.g. - 20260722 or 20260722.1). Auto-detected from git - tag alongside --localversion; pass it explicitly - whenever --localversion is passed explicitly. + 20260722 or 20260722.1). Derived alongside + --localversion; pass it explicitly whenever + --localversion is passed explicitly. --git-sha SHA Full commit SHA the build was cut from. Its first 12 characters discriminate two builds of one snapshot (a moved tag) in the version @@ -108,13 +113,14 @@ OPTIONS: -h, --help Show this help EXAMPLES: - # Minimal: auto-detect LOCALVERSION from git tag, default package names + # Minimal: derive LOCALVERSION from the checkout, default package names $0 --source-dir /path/to/kernel # Full CI invocation with all options $0 --source-dir /path/to/kernel \\ --distro trixie \\ - --localversion +qcom-next-20260722 \\ + --localversion +qcom-next-20260722-g07f50dc44edd \\ + --snapshot 20260722 \\ --srcpkg linux-qcom-next \\ --binpkg linux-image-qcom-next \\ --debian-revision 0qcom1 \\ @@ -127,6 +133,7 @@ EOF # Defaults SOURCE_DIR="" DISTRO="$DEFAULT_DISTRO" +FLAVOUR="$DEFAULT_FLAVOUR" LOCALVERSION="" SNAPSHOT="" KVER_EXTRA="" @@ -143,6 +150,7 @@ while [[ $# -gt 0 ]]; do case $1 in -s|--source-dir) SOURCE_DIR="$2"; shift 2 ;; -d|--distro) DISTRO="$2"; shift 2 ;; + --flavour) FLAVOUR="$2"; shift 2 ;; --localversion) LOCALVERSION="$2"; shift 2 ;; --snapshot) SNAPSHOT="$2"; shift 2 ;; --git-sha) GIT_SHA="$2"; shift 2 ;; @@ -180,38 +188,42 @@ VALID_DISTROS=(noble questing resolute trixie forky sid unstable) [[ -n "$GIT_SHA" ]] || GIT_SHA=$(git -C "$SOURCE_DIR" rev-parse HEAD 2>/dev/null || true) GITSHA="${GIT_SHA:0:12}" -# ── Helper: derive LOCALVERSION, SNAPSHOT and GITSHA from a tag name ───────── -# qcom-next-7.2-rc3-20260722 -> +qcom-next-20260722-g / 20260722 -# qcom-next-7.2-rc3-20260722.1 -> +qcom-next-20260722.1-g / 20260722.1 -# -# The trailing component is a YYYYMMDD snapshot with an optional respin ordinal -# for a second tag cut on the same day. Matching the date width explicitly (and -# not just "trailing digits") keeps the ordinal attached to it. -# -# All three fields come out of the tag and HEAD together. Recovering them from -# LOCALVERSION afterwards would mean parsing a string that also holds a variant -# name and a hex SHA that can end in eight digits. -_auto_version_fields() { - local tag="$1" - if [[ "$tag" =~ ^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]{8}(\.[0-9]+)?)$ ]]; then - SNAPSHOT="${BASH_REMATCH[2]}" - LOCALVERSION="+${BASH_REMATCH[1]}-${SNAPSHOT}-g${GITSHA}" - else - LOCALVERSION="+$tag" - SNAPSHOT="" - fi -} - -# ── Auto-detect LOCALVERSION, SNAPSHOT and GITSHA from git (if not provided) ── +# ── Derive LOCALVERSION, SNAPSHOT and GITSHA from the checkout (if not given) ── +# The same derivation CI performs, by the same script: the ref is the exact +# tag HEAD sits on, or the branch it is the tip of, and the date is HEAD's +# committer date for the branch-tip case. A local build of a commit therefore +# produces the version CI would give it, which is what lets a source package +# built here stand in for one built there. if [[ -z "$LOCALVERSION" ]]; then - GIT_TAG=$(git -C "$SOURCE_DIR" describe --tags --exact-match 2>/dev/null || true) - if [[ -n "$GIT_TAG" ]]; then - _auto_version_fields "$GIT_TAG" - log_info "Auto-detected LOCALVERSION='$LOCALVERSION' SNAPSHOT='$SNAPSHOT' GITSHA='$GITSHA' from tag '$GIT_TAG'" + if [[ -z "$GIT_SHA" ]]; then + log_warn "LOCALVERSION not set and $SOURCE_DIR is not a git checkout." + log_warn "Package will be named linux-image- (no flavour/date suffix)." + log_warn "Use --localversion to specify, e.g.: --localversion +qcom-next-20260722-g07f50dc44edd" else - log_warn "LOCALVERSION not set and no exact git tag found." - log_warn "Package will be named linux-image- (no branch/date suffix)." - log_warn "Use --localversion to specify, e.g.: --localversion +qcom-next-20260722" + GIT_TAG=$(git -C "$SOURCE_DIR" describe --tags --exact-match 2>/dev/null || true) + if [[ -n "$GIT_TAG" ]]; then + DERIVE_REF="$GIT_TAG" + else + # A detached HEAD reports HEAD, which derive-localversion.sh + # treats like any undated ref: the commit date supplies the + # snapshot. Committer date, normalised to UTC, as CI does. + DERIVE_REF=$(git -C "$SOURCE_DIR" rev-parse --abbrev-ref HEAD) + fi + DERIVE_DATE=$(TZ=UTC git -C "$SOURCE_DIR" log -1 --format=%cd --date=format-local:%Y%m%d) + FIELDS=$("$SCRIPT_DIR/ci/scripts/derive-localversion.sh" \ + --flavour "$FLAVOUR" \ + --ref "$DERIVE_REF" \ + --sha "$GIT_SHA" \ + --date "$DERIVE_DATE") + while IFS='=' read -r key value; do + case "$key" in + LOCALVERSION) LOCALVERSION="$value" ;; + SNAPSHOT) SNAPSHOT="$value" ;; + GITSHA) GITSHA="$value" ;; + *) log_error "Unexpected field '$key' from derive-localversion.sh"; exit 1 ;; + esac + done <<< "$FIELDS" + log_info "Derived LOCALVERSION='$LOCALVERSION' SNAPSHOT='$SNAPSHOT' GITSHA='$GITSHA' from $DERIVE_REF" fi elif [[ -z "$SNAPSHOT" ]]; then # An explicit --localversion is not parsed for a snapshot; say so rather @@ -223,6 +235,7 @@ fi log_step "Configuration:" log_info " Source dir: $SOURCE_DIR" log_info " Distro: $DISTRO" +log_info " Flavour: $FLAVOUR" log_info " Source package: $SRCPKG" log_info " Binary metapkg: $BINPKG" log_info " Debian revision: $DEBIAN_REVISION" From 3afaa93ab6e55d4d9fd365243323c921e73a20f8 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 56/71] build-kernel.sh: resolve the latest tag the way CI does --latest-tag picked the tag with the highest version sort, which puts an older-dated release candidate of a newer kernel above the newest snapshot. CI resolves by trailing date through ci/scripts/resolve-kernel-ref.sh, so a local build now calls the same script and lands on the same tag. Signed-off-by: Christopher Obbard --- build-kernel.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/build-kernel.sh b/build-kernel.sh index c3b64fc0..5cc410c5 100755 --- a/build-kernel.sh +++ b/build-kernel.sh @@ -186,13 +186,12 @@ echo # ── Git operations: resolve ref → sync → checkout ──────────────────────────── if [[ -z "$LOCAL_SOURCE" ]]; then - # Resolve the latest tag remotely before any network I/O (avoids fetching all tags) + # Resolve the latest tag remotely before any network I/O (avoids fetching + # all tags). The same script CI uses, so "latest" means the same thing + # here: the newest trailing date, not the highest kernel version. if [[ "$LATEST_TAG" == true ]]; then log_step "Finding latest qcom-next-* tag from remote..." - TAG=$(git ls-remote --tags "$REPO" 'refs/tags/qcom-next-*' \ - | awk '{print $2}' | sed 's|refs/tags/||' | grep -v '\^{}' \ - | sort -V | tail -1) - [[ -n "$TAG" ]] || { log_error "No qcom-next-* tags found in $REPO"; exit 1; } + TAG=$("$SCRIPT_DIR/ci/scripts/resolve-kernel-ref.sh" --url "$REPO" --latest-tag 'qcom-next-*') log_info "Latest tag: $TAG" fi From 6c31e209f204e6f46d49ddb0ce037916ee3fc6d4 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 57/71] prepare-source.sh: record where a local build came from The changelog names the repository and ref a build was cut from, but only CI passed them, so a local build's changelog read "Source: unknown unknown". The origin remote and the tag or branch HEAD answers to are the same facts, read from the checkout, so they are now the defaults for --git-clone and --git-ref. Naming the checkout once also serves the version derivation, which used the same tag-or-branch rule of its own. Signed-off-by: Christopher Obbard --- prepare-source.sh | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/prepare-source.sh b/prepare-source.sh index b4c69c25..6e81e778 100755 --- a/prepare-source.sh +++ b/prepare-source.sh @@ -72,8 +72,10 @@ OPTIONS: --kver-extra SUFFIX Extra suffix appended to the final KVER (e.g. -ci42). --git-clone URL Kernel repository URL, recorded in the changelog. + Defaults to the checkout's origin remote. --git-ref REF Resolved kernel ref (tag or branch), recorded in - the changelog. + the changelog. Defaults to the exact tag HEAD + sits on, or the branch it is the tip of. Package naming: --srcpkg NAME Source package name (default: $DEFAULT_SRCPKG) @@ -188,6 +190,20 @@ VALID_DISTROS=(noble questing resolute trixie forky sid unstable) [[ -n "$GIT_SHA" ]] || GIT_SHA=$(git -C "$SOURCE_DIR" rev-parse HEAD 2>/dev/null || true) GITSHA="${GIT_SHA:0:12}" +# ── Name the checkout ──────────────────────────────────────────────────────── +# The ref HEAD answers to: the exact tag it sits on, else the branch it is the +# tip of. A detached HEAD reports HEAD, which is treated like any undated ref. +# This names the checkout for the version derivation below and, when the +# caller gave none, for the provenance recorded in the changelog -- so a local +# build says where it came from just as a CI build does. +CHECKOUT_REF="" +if [[ -n "$GIT_SHA" ]]; then + CHECKOUT_REF=$(git -C "$SOURCE_DIR" describe --tags --exact-match 2>/dev/null \ + || git -C "$SOURCE_DIR" rev-parse --abbrev-ref HEAD) + [[ -n "$GIT_CLONE" ]] || GIT_CLONE=$(git -C "$SOURCE_DIR" remote get-url origin 2>/dev/null || true) + [[ -n "$GIT_REF" ]] || GIT_REF="$CHECKOUT_REF" +fi + # ── Derive LOCALVERSION, SNAPSHOT and GITSHA from the checkout (if not given) ── # The same derivation CI performs, by the same script: the ref is the exact # tag HEAD sits on, or the branch it is the tip of, and the date is HEAD's @@ -200,19 +216,11 @@ if [[ -z "$LOCALVERSION" ]]; then log_warn "Package will be named linux-image- (no flavour/date suffix)." log_warn "Use --localversion to specify, e.g.: --localversion +qcom-next-20260722-g07f50dc44edd" else - GIT_TAG=$(git -C "$SOURCE_DIR" describe --tags --exact-match 2>/dev/null || true) - if [[ -n "$GIT_TAG" ]]; then - DERIVE_REF="$GIT_TAG" - else - # A detached HEAD reports HEAD, which derive-localversion.sh - # treats like any undated ref: the commit date supplies the - # snapshot. Committer date, normalised to UTC, as CI does. - DERIVE_REF=$(git -C "$SOURCE_DIR" rev-parse --abbrev-ref HEAD) - fi + # Committer date, normalised to UTC, as CI does. DERIVE_DATE=$(TZ=UTC git -C "$SOURCE_DIR" log -1 --format=%cd --date=format-local:%Y%m%d) FIELDS=$("$SCRIPT_DIR/ci/scripts/derive-localversion.sh" \ --flavour "$FLAVOUR" \ - --ref "$DERIVE_REF" \ + --ref "$CHECKOUT_REF" \ --sha "$GIT_SHA" \ --date "$DERIVE_DATE") while IFS='=' read -r key value; do @@ -223,7 +231,7 @@ if [[ -z "$LOCALVERSION" ]]; then *) log_error "Unexpected field '$key' from derive-localversion.sh"; exit 1 ;; esac done <<< "$FIELDS" - log_info "Derived LOCALVERSION='$LOCALVERSION' SNAPSHOT='$SNAPSHOT' GITSHA='$GITSHA' from $DERIVE_REF" + log_info "Derived LOCALVERSION='$LOCALVERSION' SNAPSHOT='$SNAPSHOT' GITSHA='$GITSHA' from $CHECKOUT_REF" fi elif [[ -z "$SNAPSHOT" ]]; then # An explicit --localversion is not parsed for a snapshot; say so rather @@ -242,6 +250,7 @@ log_info " Debian revision: $DEBIAN_REVISION" [[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" [[ -n "$SNAPSHOT" ]] && log_info " SNAPSHOT: $SNAPSHOT" [[ -n "$GITSHA" ]] && log_info " GITSHA: $GITSHA" +[[ -n "$GIT_CLONE" ]] && log_info " Source: $GIT_CLONE $GIT_REF" [[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" [[ -n "$KERNEL_CONFIG" ]] && log_info " Kernel config: $KERNEL_CONFIG" [[ -n "$DKMS_MODULES" ]] && log_info " DKMS modules: $DKMS_MODULES" From 7344811003949af51cd81adb33cb22c83e6df09d Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:23 +0100 Subject: [PATCH 58/71] packaging: date debian/changelog from the kernel commit The changelog entry carried a date fixed in the template, so every build of every kernel was dated 14 November 2025. The date is now a prepare input, CHANGELOG_DATE, and prepare-source.sh fills it with HEAD's committer date: the same instant the snapshot comes from, so the entry is dated by the source it describes and two builds of one commit write one changelog. That date does more than read well. dpkg-source takes the debian tarball's mtimes from the changelog date and dpkg-buildpackage exports it as SOURCE_DATE_EPOCH, so dating the changelog from the commit is part of what makes the source package a function of the commit alone. The action reads the date on the runner and passes it in, as it does the SHA and the ref: prepare-source.sh runs in a container as root and git will not read a checkout owned by someone else. Its git lookups now tolerate that, and only run for a value the caller did not supply. A prepare run by hand without one gets today's date. Signed-off-by: Christopher Obbard --- .../actions/prepare-kernel-source/action.yml | 9 +++++ debian/changelog.in | 2 +- debian/rules | 11 ++++++ prepare-source.sh | 36 ++++++++++++++----- 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index 9e2903ae..55adb59a 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -245,6 +245,13 @@ runs: echo "kernel_date=$KERNEL_DATE" >> "$GITHUB_OUTPUT" echo "Kernel HEAD commit date (UTC): $KERNEL_DATE" + # The same instant in RFC 2822 form, for the changelog entry. Read + # here, on the host that owns the clone, because prepare-source.sh + # runs in a container as root and git will not read a checkout owned + # by someone else. + CHANGELOG_DATE=$(git -C "$KERNEL_DIR" log -1 --format=%cD) + echo "changelog_date=$CHANGELOG_DATE" >> "$GITHUB_OUTPUT" + - name: Derive version fields id: localversion shell: bash @@ -308,6 +315,7 @@ runs: KERNEL_URL: ${{ inputs.kernel-url }} KERNEL_REF: ${{ steps.ref.outputs.kernel_ref }} KERNEL_SHA: ${{ steps.clone.outputs.kernel_sha }} + CHANGELOG_DATE: ${{ steps.clone.outputs.changelog_date }} run: | # No -x here: keep xtrace off for this step as it assembles the # prepare-source.sh argument list which may include sensitive paths. @@ -319,6 +327,7 @@ runs: --srcpkg "$SRCPKG_INPUT" --binpkg "$BINPKG_INPUT" --debian-revision "$DEBIAN_REVISION" + --changelog-date "$CHANGELOG_DATE" ) # The version fields all come from derive-localversion.sh, which diff --git a/debian/changelog.in b/debian/changelog.in index 802a600a..4a67cd49 100644 --- a/debian/changelog.in +++ b/debian/changelog.in @@ -6,4 +6,4 @@ * Commit: @GITSHA@ * Includes kernel image, modules, DTBs, headers, and debug symbols - -- Bjordis Collaku Fri, 14 Nov 2025 10:00:00 -0800 + -- Bjordis Collaku @DATE@ diff --git a/debian/rules b/debian/rules index d888c1c0..9ab68b34 100755 --- a/debian/rules +++ b/debian/rules @@ -72,6 +72,15 @@ KREL_FILE := debian/kernel.release # GIT_SHA Full kernel HEAD commit SHA, recorded in the changelog. # GITSHA above is the same commit at 12 characters, for the # version strings; this is the unabbreviated value. +# CHANGELOG_DATE Date of the changelog entry, RFC 2822 (date -R form). +# prepare-source.sh passes the kernel commit's committer +# date, so the entry is dated by the source it describes +# and two builds of one commit write one changelog. +# dpkg-source takes the debian tarball's mtimes from this +# date and dpkg-buildpackage exports it as +# SOURCE_DATE_EPOCH, so it is also what makes the source +# package reproducible. Defaults to now, for a prepare run +# by hand. # # Outputs (generated files, all listed in debian/clean): # debian/control Substituted from debian/control.in @@ -101,6 +110,7 @@ DKMS_MODULES ?= GIT_CLONE ?= unknown GIT_REF ?= unknown GIT_SHA ?= unknown +CHANGELOG_DATE ?= $(shell date -R) .PHONY: prepare prepare: @@ -241,6 +251,7 @@ prepare: -e "s|@GITCLONE@|$(GIT_CLONE)|g" \ -e "s|@BRANCH@|$(GIT_REF)|g" \ -e "s|@GITSHA@|$(GIT_SHA)|g" \ + -e "s|@DATE@|$(CHANGELOG_DATE)|g" \ debian/changelog.in > debian/changelog; \ \ # ── Write runtime state files ───────────────────────────────────────────── diff --git a/prepare-source.sh b/prepare-source.sh index 6e81e778..c33b0551 100755 --- a/prepare-source.sh +++ b/prepare-source.sh @@ -76,6 +76,10 @@ OPTIONS: --git-ref REF Resolved kernel ref (tag or branch), recorded in the changelog. Defaults to the exact tag HEAD sits on, or the branch it is the tip of. + --changelog-date DATE Date of the changelog entry, RFC 2822 (date -R + form). Defaults to HEAD's committer date, so + the entry is dated by the source and the + source package is reproducible. Package naming: --srcpkg NAME Source package name (default: $DEFAULT_SRCPKG) @@ -147,6 +151,7 @@ DKMS_MODULES="" GIT_CLONE="" GIT_REF="" GIT_SHA="" +CHANGELOG_DATE="" while [[ $# -gt 0 ]]; do case $1 in @@ -159,6 +164,7 @@ while [[ $# -gt 0 ]]; do --kver-extra) KVER_EXTRA="$2"; shift 2 ;; --git-clone) GIT_CLONE="$2"; shift 2 ;; --git-ref) GIT_REF="$2"; shift 2 ;; + --changelog-date) CHANGELOG_DATE="$2"; shift 2 ;; --srcpkg) SRCPKG="$2"; shift 2 ;; --binpkg) BINPKG="$2"; shift 2 ;; --debian-revision) DEBIAN_REVISION="$2"; shift 2 ;; @@ -196,13 +202,22 @@ GITSHA="${GIT_SHA:0:12}" # This names the checkout for the version derivation below and, when the # caller gave none, for the provenance recorded in the changelog -- so a local # build says where it came from just as a CI build does. +# +# Every git call here tolerates failure. CI passes all of these values in and +# runs this script in a container as a different user from the one owning +# the checkout, where git refuses to read the repository at all; nothing it +# was given is then asked of git. +_git() { git -C "$SOURCE_DIR" "$@" 2>/dev/null || true; } CHECKOUT_REF="" -if [[ -n "$GIT_SHA" ]]; then - CHECKOUT_REF=$(git -C "$SOURCE_DIR" describe --tags --exact-match 2>/dev/null \ - || git -C "$SOURCE_DIR" rev-parse --abbrev-ref HEAD) - [[ -n "$GIT_CLONE" ]] || GIT_CLONE=$(git -C "$SOURCE_DIR" remote get-url origin 2>/dev/null || true) - [[ -n "$GIT_REF" ]] || GIT_REF="$CHECKOUT_REF" +if [[ -z "$GIT_REF" || -z "$LOCALVERSION" ]]; then + CHECKOUT_REF=$(_git describe --tags --exact-match) + [[ -n "$CHECKOUT_REF" ]] || CHECKOUT_REF=$(_git rev-parse --abbrev-ref HEAD) fi +[[ -n "$GIT_CLONE" ]] || GIT_CLONE=$(_git remote get-url origin) +[[ -n "$GIT_REF" ]] || GIT_REF="$CHECKOUT_REF" +# Committer date, like the snapshot: it describes the source rather than the +# build, so rebuilding a commit rewrites the same changelog entry. +[[ -n "$CHANGELOG_DATE" ]] || CHANGELOG_DATE=$(_git log -1 --format=%cD) # ── Derive LOCALVERSION, SNAPSHOT and GITSHA from the checkout (if not given) ── # The same derivation CI performs, by the same script: the ref is the exact @@ -211,8 +226,8 @@ fi # produces the version CI would give it, which is what lets a source package # built here stand in for one built there. if [[ -z "$LOCALVERSION" ]]; then - if [[ -z "$GIT_SHA" ]]; then - log_warn "LOCALVERSION not set and $SOURCE_DIR is not a git checkout." + if [[ -z "$GIT_SHA" || -z "$CHECKOUT_REF" ]]; then + log_warn "LOCALVERSION not set and $SOURCE_DIR is not a readable git checkout." log_warn "Package will be named linux-image- (no flavour/date suffix)." log_warn "Use --localversion to specify, e.g.: --localversion +qcom-next-20260722-g07f50dc44edd" else @@ -373,11 +388,16 @@ PREPARE_ARGS="DISTRO=$DISTRO SRCPKG=$SRCPKG BINPKG=$BINPKG DEBIAN_REVISION=$DEBI [[ -n "$GIT_CLONE" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_CLONE=$GIT_CLONE" [[ -n "$GIT_REF" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_REF=$GIT_REF" [[ -n "$GIT_SHA" ]] && PREPARE_ARGS="$PREPARE_ARGS GIT_SHA=$GIT_SHA" +# Quoted separately: an RFC 2822 date has spaces, which the string-built +# argument list above would split. Empty means "now", which debian/rules +# decides for itself. +CHANGELOG_DATE_ARG=() +[[ -n "$CHANGELOG_DATE" ]] && CHANGELOG_DATE_ARG=("CHANGELOG_DATE=$CHANGELOG_DATE") # Spaces are stripped so a list written as "kgsl, camx" stays a single make # argument; debian/rules validates the names it is given. [[ -n "$DKMS_MODULES" ]] && PREPARE_ARGS="$PREPARE_ARGS DKMS_MODULES=$(tr -d ' ' <<< "$DKMS_MODULES")" # shellcheck disable=SC2086 -make -f "$SOURCE_DIR/debian/rules" -C "$SOURCE_DIR" prepare $PREPARE_ARGS +make -f "$SOURCE_DIR/debian/rules" -C "$SOURCE_DIR" prepare $PREPARE_ARGS "${CHANGELOG_DATE_ARG[@]}" echo log_step "Source preparation complete." From e865428d10fcba6369b5881bd53da1e0c61cdc95 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 59/71] scripts: add build-source-package.sh Both build paths made the orig tarball with tar czf on a fresh clone, so every run wrote new mtimes and a new gzip header and the same upstream version got a different orig each time. Two suites sharing a source package and upstream version -- trixie and forky, differing only in Debian revision -- therefore could not share a pool, and a rerun could not reproduce what it had built. build-source-package.sh takes a tree prepare-source.sh has prepared and writes the .orig.tar.gz, .debian.tar.xz, .dsc and .changes into an output directory. The orig comes from git archive of the commit, whose entries carry the commit's timestamp and root ownership, through gzip -n, which writes no timestamp: two runs on one commit give one tarball, byte for byte. Verified here by building trixie and forky into one directory, where the second run rebuilt the orig and compared it with the first. The upstream version names the commit as ~g, so the script checks that the commit it archives is that one, and refuses a tree that differs from the commit outside debian/ -- dpkg-source would fold the difference into an automatic patch and the package would no longer describe the commit its version names. --write-fields hands the result to a caller as KEY=VALUE lines, for CI. Signed-off-by: Christopher Obbard --- build-source-package.sh | 236 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100755 build-source-package.sh diff --git a/build-source-package.sh b/build-source-package.sh new file mode 100755 index 00000000..12ba8a0a --- /dev/null +++ b/build-source-package.sh @@ -0,0 +1,236 @@ +#!/bin/bash +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +set -euo pipefail + +# Build a Debian source package from a prepared kernel tree. +# +# Takes a kernel checkout that prepare-source.sh has already injected debian/ +# into, and produces in an output directory the .orig.tar.gz, .debian.tar.xz, +# .dsc and .changes that a binary build -- local, Debusine, or sbuild in a +# container -- starts from. +# +# The orig tarball is a function of the commit and nothing else. It is written +# with git archive, which sets every entry's mtime to the commit time and its +# owner to root, and compressed with gzip -n, which writes no timestamp. Two +# runs on one commit therefore produce one tarball, byte for byte, and two +# matrix entries that share a source package and upstream version -- trixie +# and forky, differing only in Debian revision -- share one orig, which is +# what an archive holding both requires. +# +# The upstream version names the commit (the ~g field), so the archived +# commit is checked against it: an orig named for one commit must hold that +# commit's tree. The tree is also checked against the commit before the +# package is built, because dpkg-source would otherwise fold any stray file +# into an automatic patch and the package would no longer describe the +# commit its version names. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m' +log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +log_error() { echo -e "${RED}[ERROR]${NC} $*"; } +log_step() { echo -e "${BLUE}[STEP]${NC} $*"; } + +usage() { + cat </dev/null || { log_error "$tool not found; install dpkg-dev and git"; exit 1; } +done + +git -C "$SOURCE_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { + log_error "$SOURCE_DIR is not a git checkout; the orig tarball is archived from a commit." + exit 1 +} + +# ── Read the package identity from the changelog ───────────────────────────── +# The changelog is what prepare-source.sh generated and what the .dsc will be +# built from, so it is the only source of the name and version here. The sed +# filters keep the values to what Debian policy allows in a package name and +# version, as debusine-action does, so they are safe in filenames. +PKG=$(dpkg-parsechangelog -l "$SOURCE_DIR/debian/changelog" -SSource | sed 's/[^a-z0-9.+-]//g') +VER=$(dpkg-parsechangelog -l "$SOURCE_DIR/debian/changelog" -SVersion | sed 's/[^A-Za-z0-9.+~:-]//g') +[[ -n "$PKG" && -n "$VER" ]] || { log_error "Could not read Source and Version from debian/changelog"; exit 1; } +UPSTREAM_VER="${VER%-*}" +[[ "$UPSTREAM_VER" != "$VER" ]] || { + log_error "Version $VER has no Debian revision; a 3.0 (quilt) package needs one." + exit 1 +} + +# ── Resolve the commit and check it against the version ────────────────────── +[[ -n "$COMMIT" ]] || COMMIT=HEAD +COMMIT_SHA=$(git -C "$SOURCE_DIR" rev-parse --verify "${COMMIT}^{commit}") || { + log_error "Not a commit in $SOURCE_DIR: $COMMIT" + exit 1 +} + +# debian/rules spells the commit into the upstream version as ~g<12 hex>. An +# orig named for that commit must hold that commit's tree, so the two are +# checked against each other rather than trusted to agree. A version without +# the field (an explicit --localversion with no snapshot) names no commit, in +# which case there is nothing to check and the tarball is only as +# reproducible as the caller's choice of commit. +if [[ "$UPSTREAM_VER" =~ ~g([0-9a-f]{12}) ]]; then + VERSION_SHA="${BASH_REMATCH[1]}" + [[ "$COMMIT_SHA" == "$VERSION_SHA"* ]] || { + log_error "Version $VER names commit $VERSION_SHA, but $COMMIT is $COMMIT_SHA." + log_error "Re-run prepare-source.sh on this checkout, or pass --commit $VERSION_SHA." + exit 1 + } +else + log_warn "Version $VER names no commit; the orig tarball is archived from $COMMIT_SHA." +fi + +# ── Check the tree against the commit ──────────────────────────────────────── +# dpkg-source compares the tree with the orig, so anything outside debian/ +# that differs from the commit -- an in-tree build's objects, an edit not +# committed -- would become an automatic patch or abort the build. Refuse it +# here with the list and the command that cleans it, rather than let +# dpkg-source report it one file at a time. +log_step "Checking $SOURCE_DIR matches $COMMIT_SHA outside debian/..." +if [[ "$COMMIT_SHA" != "$(git -C "$SOURCE_DIR" rev-parse HEAD)" ]]; then + log_error "HEAD of $SOURCE_DIR is not $COMMIT; check out the commit the package names." + exit 1 +fi +STRAY=$(git -C "$SOURCE_DIR" status --porcelain --ignored -- . ':(exclude)debian' | head -20) +[[ -z "$STRAY" ]] || { + log_error "$SOURCE_DIR differs from commit $COMMIT_SHA outside debian/:" + echo "$STRAY" + log_error "The source package must describe the commit its version names." + log_error "Clean the tree first, keeping the injected packaging:" + log_error " git -C $SOURCE_DIR checkout -- . && git -C $SOURCE_DIR clean -xdf -- . ':(exclude)debian'" + exit 1 +} + +# ── Write the orig tarball ─────────────────────────────────────────────────── +mkdir -p "$OUTPUT_DIR" +OUTPUT_DIR="$(cd "$OUTPUT_DIR" && pwd)" +ORIG="$OUTPUT_DIR/${PKG}_${UPSTREAM_VER}.orig.tar.gz" +DSC="$OUTPUT_DIR/${PKG}_${VER}.dsc" +CHANGES="$OUTPUT_DIR/${PKG}_${VER}_source.changes" + +PREVIOUS_ORIG_SHA256="" +[[ -f "$ORIG" ]] && PREVIOUS_ORIG_SHA256=$(sha256sum "$ORIG" | cut -d' ' -f1) + +log_step "Writing $ORIG from commit $COMMIT_SHA..." +# --format=tar piped through gzip rather than --format=tar.gz: git's own +# gzip is also deterministic, but this spells out which compressor decides +# the bytes. The bytes depend on the gzip implementation, so every build +# that must agree on a checksum runs this in the same builder image. +git -C "$SOURCE_DIR" archive --format=tar --prefix="${PKG}-${UPSTREAM_VER}/" "$COMMIT_SHA" \ + | gzip -9n > "$ORIG.tmp" +mv "$ORIG.tmp" "$ORIG" +ORIG_SHA256=$(sha256sum "$ORIG" | cut -d' ' -f1) +log_info " sha256: $ORIG_SHA256" + +if [[ -n "$PREVIOUS_ORIG_SHA256" && "$PREVIOUS_ORIG_SHA256" != "$ORIG_SHA256" ]]; then + # The file that was there carried the same name and so claimed the same + # commit. It cannot have been the same tarball, so something about how + # it was made differs from this run -- which is what this script exists + # to rule out. + log_error "An orig tarball for $PKG $UPSTREAM_VER already existed with sha256 $PREVIOUS_ORIG_SHA256" + log_error "and rebuilding it from $COMMIT_SHA gave $ORIG_SHA256." + log_error "Two files with this name must be one file; find what changed between the two builds." + exit 1 +fi + +# ── Build the .dsc and .changes ────────────────────────────────────────────── +# dpkg-source -b writes into the current directory and finds the orig there, +# so it runs from the output directory. dpkg-genchanges reads debian/ from +# the source tree and is told where the files are and where to write. +log_step "Building the source package..." +rm -f "$DSC" "$OUTPUT_DIR/${PKG}_${VER}.debian.tar."* "$CHANGES" +(cd "$OUTPUT_DIR" && dpkg-source -b "$SOURCE_DIR") +(cd "$SOURCE_DIR" && dpkg-genchanges -S -sa -u"$OUTPUT_DIR" -O"$CHANGES" >/dev/null) + +[[ -f "$DSC" && -f "$CHANGES" ]] || { + log_error "Expected $DSC and $CHANGES to exist after the build" + exit 1 +} + +echo +log_step "Source package complete." +log_info " $CHANGES" +grep -A100 '^Checksums-Sha256:' "$CHANGES" | sed -n '2,/^[A-Z]/{/^ /p}' | sed 's/^/ /' + +if [[ -n "$FIELDS_FILE" ]]; then + cat > "$FIELDS_FILE" < Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 60/71] build-kernel.sh: build a source package, and build from one --source-package stops after build-source-package.sh has written the .orig.tar.gz, .dsc and .changes into the output directory, and --dsc builds binaries from an existing source package with no clone and no prepare. Together they let a local build follow the path CI takes: source package first, binaries from it. Native mode unpacks the .dsc into the output directory and runs dpkg-buildpackage -b there, so the .deb files land where every other mode puts them. sbuild takes a .dsc as it stands. Docker mode cannot go through docker_deb_build.py, which builds from a tree only, so it runs sbuild in the same pkg-builder image with the same flags docker_deb_build.py uses, building the image through docker_deb_build.py --rebuild if it is absent. The default path from a tree is unchanged. Signed-off-by: Christopher Obbard --- build-kernel.sh | 130 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 110 insertions(+), 20 deletions(-) diff --git a/build-kernel.sh b/build-kernel.sh index 5cc410c5..c204e87c 100755 --- a/build-kernel.sh +++ b/build-kernel.sh @@ -37,6 +37,18 @@ OPTIONS: -l, --latest-tag Select the latest qcom-next-* tag automatically -b, --branch BRANCH Branch to use (default: $DEFAULT_BRANCH) -r, --repo URL Kernel repository URL (default: $DEFAULT_REPO) + --dsc FILE Build from an existing Debian source package + instead of a kernel tree: no clone, no prepare. + The .orig.tar.gz and .debian.tar.xz the .dsc + names must sit beside it, as build-source-package.sh + and CI leave them. + + What to build: + --source-package Stop after building the Debian source package + (.orig.tar.gz, .dsc, .changes) into the output + directory, with build-source-package.sh. The orig + tarball is reproducible: see that script. Build + binaries from it later with --dsc. Version control: --flavour NAME Kernel flavour carried in LOCALVERSION @@ -86,6 +98,8 @@ EXAMPLES: $0 --local-source /path/to/kernel --kver-extra -mybuild $0 --latest-tag --kernel-config docker,systemd-boot $0 --latest-tag --dkms kgsl + $0 --latest-tag --source-package + $0 --dsc kernel-build/trixie/linux-qcom-next_*.dsc DISTRIBUTIONS: noble Ubuntu 24.04 LTS @@ -102,7 +116,7 @@ TAG=""; LATEST_TAG=false; BRANCH="$DEFAULT_BRANCH"; REPO="$DEFAULT_REPO" DISTRO="$DEFAULT_DISTRO"; BUILD_MODE="$DEFAULT_BUILD_MODE"; FLAVOUR="$DEFAULT_FLAVOUR" LOCALVERSION=""; KVER_EXTRA=""; PROFILES=""; CLEAN=false LOCAL_SOURCE=""; ENABLE_CONFIGS="squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can"; SKIP_PREPARE=false -DKMS_MODULES="" +DKMS_MODULES=""; DSC=""; SOURCE_PACKAGE=false while [[ $# -gt 0 ]]; do case $1 in @@ -115,6 +129,8 @@ while [[ $# -gt 0 ]]; do --debian-dir) DEBIAN_DIR="$2"; shift 2 ;; -d|--distro) DISTRO="$2"; shift 2 ;; --local-source) LOCAL_SOURCE="$2"; shift 2 ;; + --dsc) DSC="$2"; shift 2 ;; + --source-package) SOURCE_PACKAGE=true; shift ;; --docker-build) DOCKER_PKG_BUILD="$2"; shift 2 ;; --flavour) FLAVOUR="$2"; shift 2 ;; --localversion) LOCALVERSION="$2"; shift 2 ;; @@ -147,6 +163,17 @@ VALID_MODES=(docker native sbuild) exit 1 } +# --dsc replaces the tree: everything that selects or prepares one is +# meaningless beside it, and so is asking for the source package it already is. +if [[ -n "$DSC" ]]; then + [[ -z "$LOCAL_SOURCE$TAG" && "$LATEST_TAG" == false && "$SKIP_PREPARE" == false && "$SOURCE_PACKAGE" == false ]] || { + log_error "--dsc cannot be combined with --local-source, --tag, --latest-tag, --skip-prepare or --source-package" + exit 1 + } + [[ -f "$DSC" ]] || { log_error "Source package not found: $DSC"; exit 1; } + DSC="$(cd "$(dirname "$DSC")" && pwd)/$(basename "$DSC")" +fi + # Locate docker_deb_build.py (docker mode) if [[ "$BUILD_MODE" == "docker" && -z "$DOCKER_PKG_BUILD" ]]; then for p in "$HOME/docker-pkg-build/docker_deb_build.py" \ @@ -170,9 +197,15 @@ if [[ -n "$LOCAL_SOURCE" ]]; then fi log_step "Configuration:" -[[ -n "$LOCAL_SOURCE" ]] && log_info " Source: local ($KERNEL_DIR)" \ - || log_info " Repo: $REPO branch: $BRANCH" +if [[ -n "$DSC" ]]; then + log_info " Source: $DSC" +elif [[ -n "$LOCAL_SOURCE" ]]; then + log_info " Source: local ($KERNEL_DIR)" +else + log_info " Repo: $REPO branch: $BRANCH" +fi log_info " Output: $OUTPUT_DIR" +[[ "$SOURCE_PACKAGE" == true ]] && log_info " Building: source package only" log_info " Distro: $DISTRO mode: $BUILD_MODE" [[ "$BUILD_MODE" == "docker" ]] && log_info " Docker build: $DOCKER_PKG_BUILD" [[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" \ @@ -185,7 +218,9 @@ log_info " Distro: $DISTRO mode: $BUILD_MODE" echo # ── Git operations: resolve ref → sync → checkout ──────────────────────────── -if [[ -z "$LOCAL_SOURCE" ]]; then +if [[ -n "$DSC" ]]; then + : # No tree: the source package is the source. +elif [[ -z "$LOCAL_SOURCE" ]]; then # Resolve the latest tag remotely before any network I/O (avoids fetching # all tags). The same script CI uses, so "latest" means the same thing # here: the newest trailing date, not the highest kernel version. @@ -227,7 +262,7 @@ else log_info "Using local source as-is (skipping git checkout)" fi -cd "$KERNEL_DIR" +[[ -n "$DSC" ]] || cd "$KERNEL_DIR" # ── Source preparation ──────────────────────────────────────────────────────── # Delegates to prepare-source.sh, which is the single source of truth for @@ -236,8 +271,10 @@ cd "$KERNEL_DIR" # --localversion is passed through, and otherwise prepare-source.sh derives # it from the checkout the way CI does. # Skipped when --skip-prepare is set (CI mode: prepare-source.sh already ran -# as a dedicated prior step). -if [[ "$SKIP_PREPARE" != true ]]; then +# as a dedicated prior step), and when there is no tree to prepare. +if [[ -n "$DSC" ]]; then + : +elif [[ "$SKIP_PREPARE" != true ]]; then PREPARE_ARGS=(--source-dir "$KERNEL_DIR" --distro "$DISTRO" --debian-dir "$DEBIAN_DIR" --flavour "$FLAVOUR") [[ -n "$LOCALVERSION" ]] && PREPARE_ARGS+=(--localversion "$LOCALVERSION") @@ -257,12 +294,26 @@ else } fi -# ── Build ──────────────────────────────────────────────────────────────────── +# ── Source package ─────────────────────────────────────────────────────────── mkdir -p "$OUTPUT_DIR" +if [[ "$SOURCE_PACKAGE" == true ]]; then + log_step "Building the source package into $OUTPUT_DIR..." + "$SCRIPT_DIR/build-source-package.sh" --source-dir "$KERNEL_DIR" --output-dir "$OUTPUT_DIR" + echo + log_info "Build binaries from it with:" + log_info " $0 --dsc $OUTPUT_DIR/$(cd "$OUTPUT_DIR" && ls -- *.dsc | tail -1) --distro $DISTRO" + exit 0 +fi + +# ── Build ──────────────────────────────────────────────────────────────────── log_step "Building kernel package (mode: $BUILD_MODE)..." [[ -n "$PROFILES" ]] && log_info "Build profiles: $PROFILES" echo +# Every mode builds either the prepared tree or the source package. The +# source-package path is the one CI takes for every suite, so it is the one to +# use when a local build must produce what CI produced: same .dsc in, same +# binaries out. case "$BUILD_MODE" in docker) USE_SUDO="" @@ -272,26 +323,65 @@ case "$BUILD_MODE" in [[ $REPLY =~ ^[Yy]$ ]] || { log_error "Aborted."; exit 1; } USE_SUDO="sudo" } - BUILD_CMD=("$DOCKER_PKG_BUILD" - --skip-gbp - --no-update-check - --source-dir "$KERNEL_DIR" - --output-dir "$OUTPUT_DIR" - --distro "$DISTRO") - [[ -n "$PROFILES" ]] && BUILD_CMD+=(--profiles "$PROFILES") - ${USE_SUDO:+sudo} "${BUILD_CMD[@]}" + if [[ -n "$DSC" ]]; then + # docker_deb_build.py builds from a tree only, so the .dsc is + # handed to sbuild inside the same builder image directly, with + # the same sbuild flags docker_deb_build.py uses. The image is + # named as docker-pkg-build names it; when it is missing, + # docker_deb_build.py --rebuild builds it from its Dockerfile. + DOCKER_IMAGE="ghcr.io/qualcomm-linux/pkg-builder:$DISTRO" + ${USE_SUDO:+sudo} docker image inspect "$DOCKER_IMAGE" >/dev/null 2>&1 || { + log_info "Builder image $DOCKER_IMAGE not present; building it with docker-pkg-build..." + ${USE_SUDO:+sudo} "$DOCKER_PKG_BUILD" --no-update-check --rebuild -d "$DISTRO" + } + # The .dsc's directory is mounted read-only so sbuild can read the + # tarballs it names; results go to the output mount. + SBUILD_CMD="sbuild --chroot-mode=unshare --build-dep-resolver=aptitude" + SBUILD_CMD+=" --no-clean-source --build-dir=/workspace/output" + SBUILD_CMD+=" --host=arm64 --build=arm64 --dist=$DISTRO --no-run-lintian" + [[ -n "$PROFILES" ]] && SBUILD_CMD+=" --profiles=$PROFILES" + SBUILD_CMD+=" /workspace/source/$(basename "$DSC")" + ${USE_SUDO:+sudo} docker run --rm --privileged \ + -v "$(dirname "$DSC"):/workspace/source:ro" \ + -v "$OUTPUT_DIR:/workspace/output:Z" \ + -w /workspace/output \ + "$DOCKER_IMAGE" bash -c "$SBUILD_CMD" + else + BUILD_CMD=("$DOCKER_PKG_BUILD" + --skip-gbp + --no-update-check + --source-dir "$KERNEL_DIR" + --output-dir "$OUTPUT_DIR" + --distro "$DISTRO") + [[ -n "$PROFILES" ]] && BUILD_CMD+=(--profiles "$PROFILES") + ${USE_SUDO:+sudo} "${BUILD_CMD[@]}" + fi ;; native) log_info "Running dpkg-buildpackage on host..." [[ -n "$PROFILES" ]] && export DEB_BUILD_PROFILES="$PROFILES" - dpkg-buildpackage -us -uc -b - find "$(dirname "$KERNEL_DIR")" -maxdepth 1 -name "*.deb" -exec mv -v {} "$OUTPUT_DIR/" \; + if [[ -n "$DSC" ]]; then + # Unpack into the output directory, so dpkg-buildpackage's ../ + # is the output directory and the .deb files land there. + BUILD_TREE="$OUTPUT_DIR/$(basename "$DSC" .dsc)" + rm -rf "$BUILD_TREE" + dpkg-source -x "$DSC" "$BUILD_TREE" + (cd "$BUILD_TREE" && dpkg-buildpackage -us -uc -b) + else + dpkg-buildpackage -us -uc -b + find "$(dirname "$KERNEL_DIR")" -maxdepth 1 -name "*.deb" -exec mv -v {} "$OUTPUT_DIR/" \; + fi ;; sbuild) log_info "Running sbuild for $DISTRO..." - SBUILD_CMD=(sbuild --dist "$DISTRO" --arch arm64 --no-source) + SBUILD_CMD=(sbuild --dist "$DISTRO" --arch arm64) [[ -n "$PROFILES" ]] && SBUILD_CMD+=(--profiles "$PROFILES") - "${SBUILD_CMD[@]}" + if [[ -n "$DSC" ]]; then + # sbuild takes a .dsc directly and writes beside its cwd. + (cd "$OUTPUT_DIR" && "${SBUILD_CMD[@]}" "$DSC") + else + "${SBUILD_CMD[@]}" --no-source + fi ;; esac From 1fff653e33ade2f115425e64cb0564d12c652890 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 61/71] docs: describe building and consuming a source package locally debian/README.md now lists the two scripts beside build-kernel.sh, shows the source-package flow (--source-package, --dsc, and the two scripts run by hand), and says what makes the orig tarball reproducible and what the script refuses. It also stops claiming that build-kernel.sh derives LOCALVERSION from the tag and that debian/rules recovers it from a package name; neither has been true for a while. Signed-off-by: Christopher Obbard --- debian/README.md | 63 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/debian/README.md b/debian/README.md index 9a5380c8..0863f55c 100644 --- a/debian/README.md +++ b/debian/README.md @@ -49,6 +49,8 @@ for dependency resolution. ``` pkg-linux-qcom/ ├── build-kernel.sh ← Build orchestrator (clone → prepare → build) +├── prepare-source.sh ← Inject debian/, derive the version, run 'prepare' +├── build-source-package.sh ← Prepared tree → reproducible .orig.tar.gz, .dsc, .changes ├── debian/ │ ├── control.in ← Source-of-truth template (version-controlled) │ ├── changelog.in ← Source-of-truth template (version-controlled) @@ -184,17 +186,14 @@ KVER = + LOCALVERSION passed via `--localversion`) 3. Optionally appending `KVER_EXTRA` (passed via `--kver-extra`) -The LOCALVERSION suffix encodes the branch name and snapshot/ABI date. For -tagged builds, `build-kernel.sh` auto-extracts this from the tag name. +The LOCALVERSION suffix encodes the flavour, the snapshot date and the +commit. `prepare-source.sh` derives it from the checked-out tag or branch +with `ci/scripts/derive-localversion.sh`, the same script CI uses, so a local +build of a commit gets the version CI would give it. See +[docs/version.md](../docs/version.md) for the format. -`debian/rules` recovers LOCALVERSION from the package name at build time by -stripping the `linux-image-` prefix and the trailing `-qcom` flavour suffix: - -``` -linux-image-7.2.0-qcom-next-20260826 -→ strip "linux-image-7.0.0-rc2" → -qcom-next-20260826-qcom -→ strip "-qcom" suffix → -qcom-next-20260826 (= LOCALVERSION) -``` +`debian/rules` reads LOCALVERSION back at build time from `debian/localversion`, +which `prepare` writes, rather than recovering it from a package name. --- @@ -274,6 +273,50 @@ python3 docker_deb_build.py \ `build-kernel.sh` calls this automatically after running `prepare`. +### Building a source package + +CI does not build binaries from a tree. It builds a Debian source package +first and hands that to the builder — Debusine for the Debian suites, sbuild +in the builder container for Ubuntu — so the thing that gets built is a +`.dsc` whose checksums are recorded in a `.changes`. The same is available +locally: + +```bash +# Clone, prepare, and stop after the source package (no kernel build) +./build-kernel.sh --latest-tag --source-package +# -> kernel-build/trixie/linux-qcom-next_.orig.tar.gz +# kernel-build/trixie/linux-qcom-next_-.dsc +# kernel-build/trixie/linux-qcom-next_-_source.changes + +# Build binaries from it, in any mode +./build-kernel.sh --dsc kernel-build/trixie/linux-qcom-next_*.dsc +./build-kernel.sh --dsc kernel-build/trixie/linux-qcom-next_*.dsc --build-mode native + +# Or step by step, from a prepared tree +./prepare-source.sh --source-dir kernel-source --distro trixie +./build-source-package.sh --source-dir kernel-source --output-dir kernel-build/trixie +``` + +The `.orig.tar.gz` is a function of the kernel commit and nothing else. It +is written with `git archive`, which gives every entry the commit's timestamp +and root ownership, and compressed with `gzip -n`, which writes no timestamp, +so two builds of one commit produce one tarball byte for byte — and so do +two suites that differ only in Debian revision, which share the tarball an +archive requires them to share. `build-source-package.sh` checks that the +commit it archives is the one the version names (`~g`), and refuses a +tree that differs from that commit outside `debian/`, since `dpkg-source` +would otherwise fold the difference into an automatic patch. An in-tree +build leaves such differences; clean them with + +```bash +git -C kernel-source checkout -- . && git -C kernel-source clean -xdf -- . ':(exclude)debian' +``` + +The debian tarball and `.dsc` are reproducible too: the changelog is dated +from the commit, and `dpkg-source` takes the tarball's timestamps from it. +Only the `gzip` implementation is outside the commit's control, which is why +CI writes every orig in one builder image. + --- ## Packages produced From 170691f5b1df527b571820544b7ffb2e88a906b6 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 62/71] ci: build the source package in prepare prepare-kernel-source now runs build-source-package.sh after prepare-source.sh, in the same pkg-builder container, and uploads the resulting .changes set as source-package--. Nothing consumes it yet: the prepared-tree artifact is still uploaded and still what both build paths read, so this commit changes what a run produces and not what it builds. The next commits move each family onto it. The version the caller is told now comes from the fields file the script writes, read from the .dsc it built, rather than from a sed over the changelog on the runner host. The action also reports the orig tarball's name and SHA-256, which a caller can hold against what an archive already has for this upstream version. The container is told to trust the checkout, which the runner owns and root inside the container would otherwise refuse to read; it is this job's own clone. Signed-off-by: Christopher Obbard --- .../actions/prepare-kernel-source/action.yml | 95 +++++++++++++++---- 1 file changed, 74 insertions(+), 21 deletions(-) diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index 55adb59a..116c414d 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -1,7 +1,7 @@ name: Prepare kernel source description: > - Resolve the kernel ref, clone it, inject debian/ and generate the source - package, then upload the prepared tree as an artifact. + Resolve the kernel ref, clone it, inject debian/, build the Debian source + package, and upload it as an artifact alongside the prepared tree. # Every build takes this path, whichever family it belongs to, so it lives in a # composite action rather than a reusable workflow: the two family workflows @@ -21,13 +21,25 @@ description: > # come from the build matrix as the pull request wrote it. outputs: + srcpkg_name: + description: 'Name of the source package this build produces' + value: ${{ steps.version.outputs.srcpkg_name }} srcpkg_version: description: > - Version of the source package this build will produce, read from the - generated debian/changelog. A caller that promotes the result into an - archive needs it before the build starts, to see whether that version is - already published there. + Version of the source package this build produces, read from the .dsc + build-source-package.sh wrote. A caller that promotes the result into + an archive needs it before the build starts, to see whether that + version is already published there. value: ${{ steps.version.outputs.srcpkg_version }} + orig_file: + description: 'Filename of the .orig.tar.gz the source package references' + value: ${{ steps.version.outputs.orig_file }} + orig_sha256: + description: > + SHA-256 of that orig tarball. It is a function of the kernel commit, so + a caller can compare it with what an archive already holds for this + upstream version and fail on a difference. + value: ${{ steps.version.outputs.orig_sha256 }} inputs: build: @@ -352,28 +364,69 @@ runs: "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" \ ./prepare-source.sh "${ARGS[@]}" - - name: Read the generated source version - # prepare-source.sh has just written debian/changelog, so the version - # this build will produce is now decided and can be reported to the - # caller. Taken from the changelog rather than recomputed here: that file - # is what the source package is built from, so nothing can disagree with - # it. Parsed with sed rather than dpkg-parsechangelog because these steps - # run on the runner host, which is not required to have dpkg-dev; the - # first line of a changelog is "srcpkg (version) suite; urgency=...". - id: version + - name: Build source package + # The .orig.tar.gz, .dsc and .changes the binary build starts from, + # whichever family builds it. Built here, in the same container that + # ran prepare-source.sh, because the runner host has no dpkg-dev and + # because the orig's bytes depend on the gzip that wrote them: every + # build that must agree on a checksum writes its orig in this image. + # + # git is told to trust the checkout, which the runner owns and the + # container reads as root; the checkout is this job's own clone. The + # fields file lands beside the output directory, outside the artifact. shell: bash env: KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} + DOCKER_DISTRO: ${{ steps.env.outputs.docker_distro }} + run: | + set -euo pipefail + docker run -i --rm \ + -v "$PWD:$PWD" --workdir="$PWD" \ + -e GIT_CONFIG_COUNT=1 \ + -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0='*' \ + "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" \ + ./build-source-package.sh \ + --source-dir "$KERNEL_DIR" \ + --output-dir "$PWD/source-package" \ + --write-fields "$PWD/source-package.fields" + + - name: Read the source package fields + # build-source-package.sh has decided the version and written it into + # the .dsc, so what it reports is what will be built and nothing can + # disagree with it. Read from its fields file rather than recomputed: + # these steps run on the runner host, which has no dpkg-dev. The values + # are the script's own, filtered to the characters Debian allows in a + # name and version, so none can carry a newline into GITHUB_OUTPUT. + id: version + shell: bash run: | set -euo pipefail - SRCPKG_VERSION=$(sed -n '1s/^[^ ]* (\([^)]*\)).*/\1/p' \ - "$KERNEL_DIR/debian/changelog") - [[ -n "$SRCPKG_VERSION" ]] || { - echo "::error::Could not read a version from the generated debian/changelog." + while IFS='=' read -r key value; do + case "$key" in + SRCPKG_NAME) echo "srcpkg_name=$value" ;; + SRCPKG_VERSION) echo "srcpkg_version=$value" ;; + ORIG) echo "orig_file=$(basename "$value")" ;; + ORIG_SHA256) echo "orig_sha256=$value" ;; + UPSTREAM_VERSION|CHANGES|DSC) ;; + *) echo "::error::unexpected field '$key' from build-source-package.sh"; exit 1 ;; + esac + done < source-package.fields >> "$GITHUB_OUTPUT" + grep -q '^SRCPKG_VERSION=.' source-package.fields || { + echo "::error::build-source-package.sh reported no version." exit 1 } - echo "Source version: $SRCPKG_VERSION" - echo "srcpkg_version=$SRCPKG_VERSION" >> "$GITHUB_OUTPUT" + echo "Source package:" + cat source-package.fields + + - name: Upload source package artifact + # The whole .changes set: .changes, .dsc, .debian.tar.xz and the orig. + # Plain files with no execute bits to lose, so no tarball around them. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: source-package-${{ inputs.flavour }}-${{ inputs.suite }} + path: source-package/ + if-no-files-found: error + retention-days: 7 - name: Pack prepared source tree shell: bash From f1a29ec91cc53a0bc82a9fc2f91a9ed946e860d0 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 63/71] ci: hand Debusine the source package prepare built The build job downloaded the prepared tree, unpacked it, and had debusine-action's generate-source-package assemble an orig tarball from it with tar czf -- a different tarball on every run. It now downloads the .changes set the prepare job built, reads the version from the .dsc, and gives lib/build that .dsc: the source package Debusine builds is the one prepare-kernel-source wrote and reported, with the reproducible orig. DEBUSINE_ASSEMBLE_ORIG in debusine-action stays for its other callers; nothing here uses it any more. Signed-off-by: Christopher Obbard --- .github/actions/debusine-build/action.yml | 46 +++++++++++++---------- .github/workflows/build-kernel-debian.yml | 4 +- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.github/actions/debusine-build/action.yml b/.github/actions/debusine-build/action.yml index 86a7404e..7e888b67 100644 --- a/.github/actions/debusine-build/action.yml +++ b/.github/actions/debusine-build/action.yml @@ -1,7 +1,7 @@ name: Build in Debusine description: > - Generate the Debian source package from the prepared tree and submit it to - Debusine, returning the CI workspace it was built in. + Submit the source package the prepare job built to Debusine, returning the + CI workspace it was built in. # Called by build-kernel-debian.yml, which decides what becomes of the result: # every build is downloaded and uploaded to S3, and one whose caller named a @@ -56,8 +56,8 @@ outputs: description: 'Browser URL of that workspace' value: ${{ steps.build-debusine.outputs.workspace_url }} srcpkg_version: - description: 'Version of the generated source package' - value: ${{ steps.generate-srcpkg.outputs.srcpkg_version }} + description: 'Version of the submitted source package' + value: ${{ steps.srcpkg.outputs.srcpkg_version }} runs: using: composite @@ -73,27 +73,33 @@ runs: sparse-checkout: | lib - - name: Download prepared source tree + # Into the workspace root: lib/build takes the .dsc from its working + # directory, and reaches its own helpers by a path relative to it, so the + # two have to be the same directory. + - name: Download source package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} - path: /tmp/srcpkg-artifact + name: source-package-${{ inputs.flavour }}-${{ inputs.suite }} - - name: Extract prepared source tree + - name: Read the source package version + # From the .dsc as downloaded, which is what Debusine will be given; + # the prepare job reported the same value from the same file. + id: srcpkg shell: bash run: | - mkdir srcpkg - tar xzf /tmp/srcpkg-artifact/*.tar.gz \ - -C srcpkg --strip-components=1 - - - name: Generate source package - id: generate-srcpkg - shell: bash - env: - SUITE: ${{ inputs.suite }} - DEBUSINE_ASSEMBLE_ORIG: "true" - run: | - SUITE="$SUITE" debusine-action/lib/generate-source-package + set -euo pipefail + DSC=$(ls -- *.dsc) + [[ $(wc -l <<<"$DSC") -eq 1 ]] || { + echo "::error::Expected exactly one .dsc in the source package artifact, found: $DSC" + exit 1 + } + SRCPKG_VERSION=$(sed -n 's/^Version: //p' "$DSC") + [[ "$SRCPKG_VERSION" =~ ^[A-Za-z0-9.+~:-]+$ ]] || { + echo "::error::Could not read a version from $DSC (got '$SRCPKG_VERSION')" + exit 1 + } + echo "Submitting $DSC ($SRCPKG_VERSION)" + echo "srcpkg_version=$SRCPKG_VERSION" >> "$GITHUB_OUTPUT" - name: Build in Debusine id: build-debusine diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 94bbe49b..2f3215b3 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -22,7 +22,7 @@ name: build-kernel-debian # something else, and nothing rebuilds a ref that has already been built. # # Jobs: -# prepare (self-hosted): clone, patch, inject debian/, generate the source, +# prepare (self-hosted): clone, inject debian/, build the source package, # and decide whether the version it produces is already published. # build (debusine-pkg-builder container): submit to Debusine. # publish (self-hosted): download the .deb files and upload them to S3. @@ -284,7 +284,7 @@ jobs: srcpkg_version: ${{ steps.debusine.outputs.srcpkg_version }} steps: # Only for the debusine-build action itself: the package being built - # comes from the prepared-source artifact, not from this checkout. + # comes from the source-package artifact, not from this checkout. - name: Checkout pkg-linux-qcom uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: From c09362b76bc118b0d7a98293c8f62175507ecba5 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 64/71] ci: build the Ubuntu package from the source package The build job downloaded the prepared tree and ran build-kernel.sh --local-source --skip-prepare, which had docker_deb_build.py make an orig tarball of its own with tar czf before handing sbuild a .dsc. It now downloads the .changes set prepare built and passes its .dsc to build-kernel.sh --dsc, which runs sbuild on it in the suite's pkg-builder image. The Ubuntu family therefore builds the same kind of thing Debusine builds for the Debian family: the source package the run reported, with the reproducible orig, and nothing is prepared twice. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-ubuntu.yml | 38 ++++++++++------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index c8dfe5e1..b6f7bce6 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -19,7 +19,7 @@ name: build-kernel-ubuntu # in ci/build-matrix.yaml, rather than in a dispatch form that drifts from it. # # Jobs: -# prepare (self-hosted): clone, patch, inject debian/, generate the source. +# prepare (self-hosted): clone, inject debian/, build the source package. # build (self-hosted): build the .deb files and upload them to S3. on: @@ -122,7 +122,7 @@ jobs: kver-extra: ${{ inputs.kver-extra }} # --------------------------------------------------------------------------- - # build: build the prepared source in a suite-matched container, then publish. + # build: build the source package in a suite-matched container, then publish. # # Named for where the build happens, as the Debian workflows' Build package # (Debusine) is: this one runs in a suite-matched pkg-builder container on @@ -178,34 +178,28 @@ jobs: esac docker image ls - - name: Download prepared source tree + - name: Download source package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} - path: /tmp/srcpkg-artifact - - - name: Extract prepared source tree - run: | - # tar.gz preserves the execute bits that upload-artifact's zip would - # strip (see README); --strip-components=1 drops the archive's top dir. - mkdir -p kernel-source - tar xzf /tmp/srcpkg-artifact/*.tar.gz \ - -C kernel-source --strip-components=1 - echo "KERNEL_DIR=$GITHUB_WORKSPACE/kernel-source" >> "$GITHUB_ENV" + name: source-package-${{ inputs.flavour }}-${{ inputs.suite }} + path: source-package - name: Build kernel package run: | - # Kernel source was cloned, patched, and prepared by the prepare job. - # Pass --local-source and --skip-prepare so build-kernel.sh goes - # directly to the build step. debian/control, debian/changelog, and - # all config fragments are already baked into the extracted source - # tree; no re-derivation of LOCALVERSION, kver-extra, or debug config - # is needed here. + # The prepare job built the source package; --dsc hands it to + # sbuild in the suite's pkg-builder image, so what is built here + # is the .dsc the run reported, exactly as Debusine builds it for + # the Debian family. Nothing is cloned or prepared again. + set -euo pipefail + DSC=$(ls source-package/*.dsc) + [[ $(wc -l <<<"$DSC") -eq 1 ]] || { + echo "::error::Expected exactly one .dsc in the source package artifact, found: $DSC" + exit 1 + } ./build-kernel.sh \ --build-mode docker \ --distro "$DISTRO" \ - --local-source "$KERNEL_DIR" \ - --skip-prepare \ + --dsc "$DSC" \ --docker-build "$GITHUB_WORKSPACE/docker-pkg-build/docker_deb_build.py" - name: Upload .deb packages to S3 From 371b4019858e14221e3cacc47e24a0d99d684d5a Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 65/71] ci: stop uploading the prepared tree Both build paths now start from the source-package artifact, so the tarball of the prepared tree has no reader. It was the larger of the two uploads, and the reason the README explained why an artifact had to be a tar.gz: the tree carried execute bits that upload-artifact's zip would strip. A .changes set is plain files, so that note goes with it, and the pipeline diagrams now show the source package as the handoff. Signed-off-by: Christopher Obbard --- .../actions/prepare-kernel-source/action.yml | 19 +--------- README.md | 35 ++++++++++--------- 2 files changed, 20 insertions(+), 34 deletions(-) diff --git a/.github/actions/prepare-kernel-source/action.yml b/.github/actions/prepare-kernel-source/action.yml index 116c414d..fe6dd19d 100644 --- a/.github/actions/prepare-kernel-source/action.yml +++ b/.github/actions/prepare-kernel-source/action.yml @@ -1,7 +1,7 @@ name: Prepare kernel source description: > Resolve the kernel ref, clone it, inject debian/, build the Debian source - package, and upload it as an artifact alongside the prepared tree. + package, and upload it as an artifact for the build job to consume. # Every build takes this path, whichever family it belongs to, so it lives in a # composite action rather than a reusable workflow: the two family workflows @@ -428,20 +428,3 @@ runs: if-no-files-found: error retention-days: 7 - - name: Pack prepared source tree - shell: bash - env: - KERNEL_DIR: ${{ steps.clone.outputs.kernel_dir }} - run: | - set -euo pipefail - tar czf /tmp/kernel-srcpkg.tar.gz \ - -C "$(dirname "$KERNEL_DIR")" \ - "$(basename "$KERNEL_DIR")" - - - name: Upload prepared source tree artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: kernel-srcpkg-${{ inputs.flavour }}-${{ inputs.suite }} - path: /tmp/kernel-srcpkg.tar.gz - if-no-files-found: error - retention-days: 7 diff --git a/README.md b/README.md index f3d33cdf..556d8f2b 100644 --- a/README.md +++ b/README.md @@ -299,7 +299,7 @@ The build workflows pass their entry's own `debian_revision` through, and the `prepare-kernel-source` action requires it: there is no caller without an entry in hand, so there is no fallback to look one up. -Each entry has a distinct prepared-source artifact, Debusine child workspace, +Each entry has a distinct source-package artifact, Debusine child workspace, and S3 path keyed by `flavour + suite`. This prevents two flavours that both build, for example, `trixie` from consuming or publishing each other's inputs or outputs. They key on `flavour`, not on the build's `name`, so renaming a @@ -415,24 +415,29 @@ flowchart LR K["Matrix-selected kernel repository\nlatest tag, branch tip, or pinned ref"] --> PS M["pkg-linux-qcom\ndebian/ and ci/ from this commit"] --> PS - PS["prepare-source.sh\n\nInject debian/\nApply all config-available fragments plus any extras\nGenerate control, changelog, localversion, pkgversion"] --> TAR - TAR["tar czf kernel-srcpkg-variant-suite.tar.gz\nPreserves execute permissions"] --> ART - ART["GitHub Actions artifact\nOne prepared source tree per variant + suite"] + PS["prepare-source.sh\n\nInject debian/\nApply all config-available fragments plus any extras\nGenerate control, changelog, localversion, pkgversion"] --> BSP + BSP["build-source-package.sh\n\ngit archive the commit → .orig.tar.gz\ndpkg-source -b → .dsc, .debian.tar.xz\ndpkg-genchanges -S → .changes"] --> ART + ART["GitHub Actions artifact\nsource-package-variant-suite\nOne .changes set per variant + suite"] ``` -> **Why `tar.gz`?** `actions/upload-artifact` uses zip internally, which strips -> Unix execute bits. Kernel build scripts require those permissions. The tar -> archive preserves them between the prepare and build jobs. +Both steps run in the pkg-builder container on the self-hosted runner. The +orig tarball is a function of the kernel commit: `git archive` gives every +entry the commit's timestamp and root ownership, and `gzip -n` writes no +timestamp, so two runs on one commit write one tarball and two suites that +differ only in Debian revision share it. `build-source-package.sh` checks +that the commit it archives is the one the version names (`~g`) and +that the tree matches that commit outside `debian/`. See +[debian/README.md](debian/README.md#building-a-source-package) for the +local equivalent. ### Debian build path ```mermaid flowchart LR - ART["kernel-srcpkg-variant-suite\nartifact"] --> GSP + ART["source-package-variant-suite\nartifact"] --> SUBMIT subgraph source[GitHub build job: debusine-pkg-builder container] - GSP["generate-source-package\nDEBUSINE_ASSEMBLE_ORIG=true\n\nCreate .orig.tar.gz\nRun dpkg-buildpackage -S\nProduce .dsc"] --> SUBMIT - SUBMIT["lib/build\nCreate CI child workspace\nSubmit source package to Debusine"] + SUBMIT["lib/build\nCreate CI child workspace\nSubmit the .dsc to Debusine"] end SUBMIT --> DEB["Debusine\nBuild binary packages"] @@ -453,19 +458,17 @@ flowchart LR ```mermaid flowchart LR - ART["kernel-srcpkg-variant-suite\nartifact"] --> EXT + ART["source-package-variant-suite\nartifact"] --> BK subgraph build[Ubuntu build job] - EXT["Extract prepared source tree\n--strip-components=1"] --> BK - BK["build-kernel.sh\n--skip-prepare\n--local-source\n--build-mode docker\ndpkg-buildpackage -b"] --> S3 + BK["build-kernel.sh --dsc\n--build-mode docker\nsbuild in the suite's pkg-builder image"] --> S3 end S3["S3\npackage artifacts"] ``` -`--skip-prepare` is safe because `prepare-source.sh` has already generated the -packaging metadata and applied the config fragments before the artifact is -created. +Nothing is cloned or prepared here: the `.dsc` the prepare job reported is +what sbuild builds, as it is what Debusine builds on the Debian path. ## Packages From fecd9b00fc1d249248d3d090cd915259174cb844 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 66/71] ci: fail when the archive holds a different orig for this version The orig tarball is now a function of the kernel commit, so the archive can hold a different one for the same upstream version only if something about how it was written differs -- and that is what this check is for. The already-published step, which already reads the target workspace's source index, now also finds every stanza naming this run's orig and compares its SHA-256 with the one prepare reported. A mismatch fails the run with both checksums, before anything is built, rather than surfacing as a file conflict at promotion or, worse, not at all. The case that matters is the packaging rebuild: 0qli1 to 0qli2 of one snapshot, where the archive requires the two revisions to share one file. The check reads this suite's index only, since apt indexes are per suite; across suites the reproducibility of the tarball itself is the guarantee. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 30 +++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 2f3215b3..377c698e 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -224,6 +224,8 @@ jobs: SUITE: ${{ inputs.suite }} SRCPKG: ${{ inputs.srcpkg }} SRCPKG_VERSION: ${{ steps.prepare.outputs.srcpkg_version }} + ORIG_FILE: ${{ steps.prepare.outputs.orig_file }} + ORIG_SHA256: ${{ steps.prepare.outputs.orig_sha256 }} run: | # No -x: DEBUSINE_TOKEN is in env, keep xtrace off. set -euo pipefail @@ -247,8 +249,32 @@ jobs: exit 0 fi - if chdist apt-cache target-workspace showsrc "$SRCPKG" 2>/dev/null \ - | grep -qxF "Version: $SRCPKG_VERSION"; then + SOURCES=$(chdist apt-cache target-workspace showsrc "$SRCPKG" 2>/dev/null || true) + + # The orig this run wrote is a function of the kernel commit, and + # the archive may hold it already: under this version on a rerun, + # or under another Debian revision of the same upstream version -- + # a packaging rebuild, which is exactly the case where the archive + # requires the two revisions to share one file. Every stanza that + # names this orig must therefore carry this checksum. One that does + # not is a build that would be rejected at promotion for a file + # conflict, or worse, accepted somewhere that does not check; so it + # fails here, with both checksums, before anything is built. + # + # This suite only: apt indexes are per suite, so an orig published + # under another suite of the workspace is not seen here. The + # reproducibility of the tarball is what covers that case. + ARCHIVE_SHA256=$(awk -v orig="$ORIG_FILE" \ + '$1 ~ /^[0-9a-f]{64}$/ && $3 == orig { print $1 }' <<<"$SOURCES" | sort -u) + if [[ -n "$ARCHIVE_SHA256" && "$ARCHIVE_SHA256" != "$ORIG_SHA256" ]]; then + echo "::error::$DEBUSINE_WORKSPACE $SUITE already holds $ORIG_FILE with sha256 $ARCHIVE_SHA256, but this run built it as $ORIG_SHA256." + echo "::error::An orig tarball is a function of the commit it names; find what differs between the two builds before publishing either." + exit 1 + fi + [[ -z "$ARCHIVE_SHA256" ]] \ + || echo "$ORIG_FILE in $DEBUSINE_WORKSPACE $SUITE matches this build ($ORIG_SHA256)." + + if grep -qxF "Version: $SRCPKG_VERSION" <<<"$SOURCES"; then echo "$SRCPKG $SRCPKG_VERSION is already in $DEBUSINE_WORKSPACE." echo "Nothing new to build; the rest of this run is skipped." \ >> "$GITHUB_STEP_SUMMARY" From 83a24470bde5693746ba1ac1a21378e506be547e Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 67/71] build-kernel.sh: need docker-pkg-build only to build a missing image Docker mode required docker_deb_build.py up front, because building from a tree goes through it. Building from a .dsc runs sbuild in the image directly and needs the script only to build the image when it is absent, so that path now looks for it without insisting, and says what to do if the image is missing and the script is too. A run from a .dsc also stops printing the tree-preparation settings it has no use for. Signed-off-by: Christopher Obbard --- build-kernel.sh | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/build-kernel.sh b/build-kernel.sh index c204e87c..a0dd5668 100755 --- a/build-kernel.sh +++ b/build-kernel.sh @@ -174,20 +174,24 @@ if [[ -n "$DSC" ]]; then DSC="$(cd "$(dirname "$DSC")" && pwd)/$(basename "$DSC")" fi -# Locate docker_deb_build.py (docker mode) +# Locate docker_deb_build.py (docker mode). Building binaries from a tree goes +# through it, so it must exist for that. Building from a .dsc runs sbuild in +# the image directly and only needs it to build the image when the image is +# missing, and --source-package builds no binaries at all, so for those it is +# looked for and not required. if [[ "$BUILD_MODE" == "docker" && -z "$DOCKER_PKG_BUILD" ]]; then for p in "$HOME/docker-pkg-build/docker_deb_build.py" \ "$SCRIPT_DIR/docker-pkg-build/docker_deb_build.py" \ "$(which docker_deb_build.py 2>/dev/null || true)"; do [[ -x "$p" ]] && { DOCKER_PKG_BUILD="$p"; break; } done - [[ -z "$DOCKER_PKG_BUILD" ]] && { + [[ -z "$DOCKER_PKG_BUILD" && -z "$DSC" && "$SOURCE_PACKAGE" == false ]] && { log_error "docker_deb_build.py not found. Use --docker-build, set DOCKER_PKG_BUILD, or use --build-mode native." exit 1 } fi [[ "$BUILD_MODE" == "docker" && -d "$DOCKER_PKG_BUILD" ]] && DOCKER_PKG_BUILD="$DOCKER_PKG_BUILD/docker_deb_build.py" -[[ "$BUILD_MODE" == "docker" && ! -x "$DOCKER_PKG_BUILD" ]] && { log_error "Not executable: $DOCKER_PKG_BUILD"; exit 1; } +[[ "$BUILD_MODE" == "docker" && -n "$DOCKER_PKG_BUILD" && ! -x "$DOCKER_PKG_BUILD" ]] && { log_error "Not executable: $DOCKER_PKG_BUILD"; exit 1; } # Handle local source if [[ -n "$LOCAL_SOURCE" ]]; then @@ -207,14 +211,17 @@ fi log_info " Output: $OUTPUT_DIR" [[ "$SOURCE_PACKAGE" == true ]] && log_info " Building: source package only" log_info " Distro: $DISTRO mode: $BUILD_MODE" -[[ "$BUILD_MODE" == "docker" ]] && log_info " Docker build: $DOCKER_PKG_BUILD" -[[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" \ - || log_info " Flavour: $FLAVOUR" -[[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" +[[ "$BUILD_MODE" == "docker" && -n "$DOCKER_PKG_BUILD" ]] && log_info " Docker build: $DOCKER_PKG_BUILD" [[ -n "$PROFILES" ]] && log_info " Profiles: $PROFILES" -[[ -n "$ENABLE_CONFIGS" ]] && log_info " Extra configs: $ENABLE_CONFIGS" -[[ -n "$DKMS_MODULES" ]] && log_info " DKMS modules: $DKMS_MODULES" -[[ "$SKIP_PREPARE" == true ]] && log_info " Skip prepare: yes (source already prepared by prepare-source.sh)" +# The rest describes preparing a tree, which a .dsc has already had done. +if [[ -z "$DSC" ]]; then + [[ -n "$LOCALVERSION" ]] && log_info " LOCALVERSION: $LOCALVERSION" \ + || log_info " Flavour: $FLAVOUR" + [[ -n "$KVER_EXTRA" ]] && log_info " KVER_EXTRA: $KVER_EXTRA" + [[ -n "$ENABLE_CONFIGS" ]] && log_info " Extra configs: $ENABLE_CONFIGS" + [[ -n "$DKMS_MODULES" ]] && log_info " DKMS modules: $DKMS_MODULES" + [[ "$SKIP_PREPARE" == true ]] && log_info " Skip prepare: yes (source already prepared by prepare-source.sh)" +fi echo # ── Git operations: resolve ref → sync → checkout ──────────────────────────── @@ -331,6 +338,11 @@ case "$BUILD_MODE" in # docker_deb_build.py --rebuild builds it from its Dockerfile. DOCKER_IMAGE="ghcr.io/qualcomm-linux/pkg-builder:$DISTRO" ${USE_SUDO:+sudo} docker image inspect "$DOCKER_IMAGE" >/dev/null 2>&1 || { + [[ -n "$DOCKER_PKG_BUILD" ]] || { + log_error "Builder image $DOCKER_IMAGE is not present, and docker_deb_build.py was not found to build it." + log_error "Pull the image, or point --docker-build at a docker-pkg-build checkout." + exit 1 + } log_info "Builder image $DOCKER_IMAGE not present; building it with docker-pkg-build..." ${USE_SUDO:+sudo} "$DOCKER_PKG_BUILD" --no-update-check --rebuild -d "$DISTRO" } From 58ef67d49ee229012969229e704ce05e046b882b Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 68/71] ci: build the Ubuntu leg in one job The prepare job and the build job ran on the same runner label, so the only thing the artifact between them did was cross a job boundary that had no other reason to exist: uploaded by one ephemeral runner to be downloaded by the next. One job now clones, prepares, builds the source package and builds the binaries from it in place. The action still uploads the .changes set, as the record of what was built, but nothing downloads it. The build job's own image-fetch step goes with it. It repeated the action's pull-or-build decision for one suite; the action makes that decision for every suite before prepare-source.sh runs, and leaves the docker-pkg-build checkout where build-kernel.sh looks for it when it had to build the image. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-ubuntu.yml | 107 +++++++--------------- README.md | 8 +- 2 files changed, 35 insertions(+), 80 deletions(-) diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index b6f7bce6..553297d1 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -18,9 +18,8 @@ name: build-kernel-ubuntu # per Ubuntu entry it selected. That also keeps every value describing a build # in ci/build-matrix.yaml, rather than in a dispatch form that drifts from it. # -# Jobs: -# prepare (self-hosted): clone, inject debian/, build the source package. -# build (self-hosted): build the .deb files and upload them to S3. +# One job, on the self-hosted runner: clone, inject debian/, build the source +# package, build the .deb files from it, and upload them to S3. on: workflow_call: @@ -88,20 +87,38 @@ permissions: jobs: # --------------------------------------------------------------------------- - # prepare: clone kernel source, inject packaging, run prepare-source.sh. + # build: clone, prepare, build the source package, build binaries from it, + # and publish them. + # + # One job rather than a prepare job and a build job, because every step + # runs on the same runner label: the source package the prepare action + # writes is consumed here from the workspace it was written into, with no + # artifact uploaded only to be downloaded again by the next job. The action + # still uploads it, as the record of what was built. + # + # Named for where the build happens, as the Debian workflow's Build package + # (Debusine) is: this one runs in a suite-matched pkg-builder container on + # the runner rather than being submitted to a build service. # --------------------------------------------------------------------------- - prepare: - name: Generate source package + build: + name: Build package (Docker) runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] permissions: contents: read # actions/checkout packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder + env: + DISTRO: ${{ inputs.suite }} + defaults: + run: + shell: bash steps: - name: Checkout pkg-linux-qcom uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + # Also pulls, or builds from a docker-pkg-build checkout, the suite's + # pkg-builder image -- the image the binary build below runs sbuild in. - name: Prepare kernel source uses: ./.github/actions/prepare-kernel-source with: @@ -121,86 +138,24 @@ jobs: localversion: ${{ inputs.localversion }} kver-extra: ${{ inputs.kver-extra }} - # --------------------------------------------------------------------------- - # build: build the source package in a suite-matched container, then publish. - # - # Named for where the build happens, as the Debian workflows' Build package - # (Debusine) is: this one runs in a suite-matched pkg-builder container on - # the runner rather than being submitted to a build service. - # --------------------------------------------------------------------------- - build: - name: Build package (Docker) - needs: prepare - runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] - permissions: - contents: read # actions/checkout - packages: read # docker pull ghcr.io/qualcomm-linux/pkg-builder - env: - DISTRO: ${{ inputs.suite }} - defaults: - run: - shell: bash - steps: - - name: Checkout pkg-linux-qcom - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Pinned to @main while native kernel-build support is still landing - # upstream in qualcomm-linux/docker-pkg-build. - - name: Checkout docker-pkg-build - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - repository: qualcomm-linux/docker-pkg-build - ref: main - path: docker-pkg-build - - - name: Fetch builder image - env: - REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # No -x: REGISTRY_TOKEN is in env. - set -euo pipefail - # Same split as prepare-kernel-source: pull what docker-pkg-build - # publishes, build what it does not. Nothing is forced here, because - # docker_deb_build.py builds the image itself when the build below - # finds it missing -- so an unpublished suite needs no step of its - # own, only the absence of a pull. - case "$DISTRO" in - resolute) - echo "$REGISTRY_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin - docker pull "ghcr.io/qualcomm-linux/pkg-builder:$DISTRO" - ;; - *) - echo "No published pkg-builder image for $DISTRO; it will be built on demand." - ;; - esac - docker image ls - - - name: Download source package - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: source-package-${{ inputs.flavour }}-${{ inputs.suite }} - path: source-package - - name: Build kernel package run: | - # The prepare job built the source package; --dsc hands it to - # sbuild in the suite's pkg-builder image, so what is built here - # is the .dsc the run reported, exactly as Debusine builds it for - # the Debian family. Nothing is cloned or prepared again. + # The source package sits where the action wrote it. --dsc hands + # it to sbuild in the suite's pkg-builder image, so what is built + # here is the .dsc the run reported, exactly as Debusine builds it + # for the Debian family. Nothing is cloned or prepared again, and + # build-kernel.sh finds the docker-pkg-build checkout the action + # leaves behind when it had to build the image itself. set -euo pipefail DSC=$(ls source-package/*.dsc) [[ $(wc -l <<<"$DSC") -eq 1 ]] || { - echo "::error::Expected exactly one .dsc in the source package artifact, found: $DSC" + echo "::error::Expected exactly one .dsc in source-package/, found: $DSC" exit 1 } ./build-kernel.sh \ --build-mode docker \ --distro "$DISTRO" \ - --dsc "$DSC" \ - --docker-build "$GITHUB_WORKSPACE/docker-pkg-build/docker_deb_build.py" + --dsc "$DSC" - name: Upload .deb packages to S3 # build-kernel.sh outputs to kernel-build// by default. diff --git a/README.md b/README.md index 556d8f2b..58c6c0c0 100644 --- a/README.md +++ b/README.md @@ -375,9 +375,9 @@ flowchart TD end subgraph build[One build workflow per leg] - C2["prepare\nprepare-kernel-source action\nClone ref, run prepare-source.sh\nSkip the run if the version is published"] + C2["prepare\nprepare-kernel-source action\nClone ref, build the source package\nSkip the run if the version is published"] C3["build\ndebusine-build action"] - C4["build\nbuild-kernel.sh in Docker"] + C4["build\nprepare-kernel-source action, then\nbuild-kernel.sh --dsc in Docker"] C5["publish\nDownload .deb files, upload to S3"] C6["promote\nlib/release into the caller's workspace"] end @@ -394,8 +394,8 @@ flowchart TD A5 --> B1 B1 --> B2 & B5 B2 --> C2 - B5 --> C2 - C2 --> C3 & C4 + B5 --> C4 + C2 --> C3 C3 --> C5 & C6 C4 --> D1 C5 --> D1 From d675ee13f93c57f71c34e680b54cffe5df287fdd Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 69/71] ci: publish the source package to S3 beside the binaries The .deb files reach S3 under one run's directory; the source package they were built from now sits in source/ beneath it, on both paths. The .changes records every checksum, the .orig.tar.gz is the exact source, and the .dsc is enough to rebuild the packages anywhere. The Actions artifact holding the same files expires after a week; this is the copy that stays with the binaries. The Debian publish job downloads the artifact for it, since the source package was built on another runner; the Ubuntu job uploads it from the workspace it was built in. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 17 +++++++++++++++++ .github/workflows/build-kernel-ubuntu.yml | 10 ++++++++++ README.md | 4 ++++ 3 files changed, 31 insertions(+) diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 377c698e..5a096278 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -396,6 +396,23 @@ jobs: path: deb-artifacts destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ + # The source package the binaries were built from, kept beside them: + # the .changes records every checksum, and the .dsc is enough to + # rebuild the packages anywhere. The Actions artifact of the same + # files expires; this does not. + - name: Download source package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: source-package-${{ inputs.flavour }}-${{ inputs.suite }} + path: source-package + + - name: Upload source package to S3 + uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 + with: + s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} + path: source-package + destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/source/ + # --------------------------------------------------------------------------- # promote: copy the built package from the ephemeral CI workspace into the # workspace the caller named, making it installable from that archive. diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 553297d1..20b2acf9 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -167,3 +167,13 @@ jobs: s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} path: kernel-build/${{ inputs.suite }} destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ + + # The source package the binaries were built from, kept beside them, + # as the Debian workflow keeps its own. Still in the workspace where + # the action wrote it. + - name: Upload source package to S3 + uses: qualcomm-linux/upload-private-artifact-action@4940e9327cd7386acdb908b61c714c079b7d754b # aws-v4 + with: + s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} + path: source-package + destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.flavour }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/source/ diff --git a/README.md b/README.md index 58c6c0c0..ac918fd0 100644 --- a/README.md +++ b/README.md @@ -313,6 +313,10 @@ S3 outputs use these layouts, where `` is /pkg/temp///// ``` +Each holds the `.deb` files, and a `source/` directory beneath it holds the +source package they were built from: the `.changes`, `.dsc`, `.debian.tar.xz` +and `.orig.tar.gz`. + The first layout is for Debian/Debusine builds; the second is for Ubuntu Docker builds. Consumers must select the intended flavour and suite. From ef3f2c41e8786b082bcc6d35c30ac5e0b4559bd9 Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:29:24 +0100 Subject: [PATCH 70/71] ci: test for a published version before comparing the orig The orig comparison ran first, so a night on which the tracked tag had not moved failed instead of skipping: the version was already published, and the orig in the archive predates git archive, so the checksums differ. That is a run with nothing to do reported as a conflict. A run that publishes nothing uploads no files, so nothing it built can conflict with anything. The already-published test therefore comes first and exits, and the orig comparison guards only a version that is actually going to be promoted. That leaves it doing the job it was added for: a packaging rebuild publishing a second Debian revision of one upstream version, where the archive requires both revisions to name one file. Signed-off-by: Christopher Obbard --- .github/workflows/build-kernel-debian.yml | 42 +++++++++++++---------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-kernel-debian.yml b/.github/workflows/build-kernel-debian.yml index 5a096278..6049370c 100644 --- a/.github/workflows/build-kernel-debian.yml +++ b/.github/workflows/build-kernel-debian.yml @@ -251,15 +251,29 @@ jobs: SOURCES=$(chdist apt-cache target-workspace showsrc "$SRCPKG" 2>/dev/null || true) - # The orig this run wrote is a function of the kernel commit, and - # the archive may hold it already: under this version on a rerun, - # or under another Debian revision of the same upstream version -- - # a packaging rebuild, which is exactly the case where the archive - # requires the two revisions to share one file. Every stanza that - # names this orig must therefore carry this checksum. One that does - # not is a build that would be rejected at promotion for a file - # conflict, or worse, accepted somewhere that does not check; so it - # fails here, with both checksums, before anything is built. + # Whether there is anything to do at all, before anything is + # inspected about what would be built: a run that publishes nothing + # uploads no files, so nothing it holds can conflict with the + # archive. + if grep -qxF "Version: $SRCPKG_VERSION" <<<"$SOURCES"; then + echo "$SRCPKG $SRCPKG_VERSION is already in $DEBUSINE_WORKSPACE." + echo "Nothing new to build; the rest of this run is skipped." \ + >> "$GITHUB_STEP_SUMMARY" + echo "already_published=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "$SRCPKG $SRCPKG_VERSION is not in $DEBUSINE_WORKSPACE yet." + echo "already_published=false" >> "$GITHUB_OUTPUT" + + # This version is going to be built and promoted, so the orig it + # carries has to agree with any the archive already holds under that + # name. It can hold one: a packaging rebuild publishes a second + # Debian revision of one upstream version, and the archive requires + # the two revisions to share the file. The orig is a function of the + # kernel commit, so a stanza naming it with another checksum means + # two builds of one commit disagreed. That is rejected at promotion + # as a file conflict, or worse accepted somewhere that does not + # check, so it fails here with both checksums instead. # # This suite only: apt indexes are per suite, so an orig published # under another suite of the workspace is not seen here. The @@ -274,16 +288,6 @@ jobs: [[ -z "$ARCHIVE_SHA256" ]] \ || echo "$ORIG_FILE in $DEBUSINE_WORKSPACE $SUITE matches this build ($ORIG_SHA256)." - if grep -qxF "Version: $SRCPKG_VERSION" <<<"$SOURCES"; then - echo "$SRCPKG $SRCPKG_VERSION is already in $DEBUSINE_WORKSPACE." - echo "Nothing new to build; the rest of this run is skipped." \ - >> "$GITHUB_STEP_SUMMARY" - echo "already_published=true" >> "$GITHUB_OUTPUT" - else - echo "$SRCPKG $SRCPKG_VERSION is not in $DEBUSINE_WORKSPACE yet." - echo "already_published=false" >> "$GITHUB_OUTPUT" - fi - # --------------------------------------------------------------------------- # build: generate the Debian source package and submit it to Debusine. # From a27bced9fed0d9a019d425c9d4f46fb85d39b7cd Mon Sep 17 00:00:00 2001 From: Christopher Obbard Date: Sat, 5 Sep 2026 02:34:09 +0100 Subject: [PATCH 71/71] wenable more builds Signed-off-by: Christopher Obbard --- ci/build-matrix.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ci/build-matrix.yaml b/ci/build-matrix.yaml index 454a5d95..1c0cfec1 100644 --- a/ci/build-matrix.yaml +++ b/ci/build-matrix.yaml @@ -96,20 +96,20 @@ builds: # TODO: added manual revision debian_revision: '0qli1+staging1' - # - name: qcom-next-resolute - # suite: resolute - # flavour: qcom-next - # git_clone: https://github.com/qualcomm-linux/kernel - # branch_or_tag: qcom-next - # ref_strategy: latest_tag - # tag_pattern: 'qcom-next-*' - # srcpkg: linux-qcom-next - # binpkg: linux-image-qcom-next - # kernel_config: [] - # # kgsl alone: camx-dkms and iris-vpu-dkms are not packaged for Ubuntu. - # dkms: - # - kgsl - # debian_revision: '0qli1~26.04.1' + - name: qcom-next-resolute + suite: resolute + flavour: qcom-next + git_clone: https://github.com/qualcomm-linux/kernel + branch_or_tag: qcom-next + ref_strategy: latest_tag + tag_pattern: 'qcom-next-*' + srcpkg: linux-qcom-next + binpkg: linux-image-qcom-next + kernel_config: [] + # kgsl alone: camx-dkms and iris-vpu-dkms are not packaged for Ubuntu. + dkms: + - kgsl + debian_revision: '0qli1~26.04.1' # # --------------------------------------------------------------------------- # # qcom-next-debug — the same kernel ref built with the in-tree debug